IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #863 from nosqlbench/my-NB5-I446

Browse Source 
nb5 - Issue 446 - Modified the export-docs command and 'build' workflow
This commit is contained in:
Jonathan Shook 2022-12-19 17:29:52 -06:00 committed by GitHub
commit c4415e6c0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
71 changed files with 197 additions and 6328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
.github/workflows/build.yml vendored
View File

@ -2,43 +2,105 @@ name: build
on: on:
push: push:
branches:
- main
pull_request: pull_request:
jobs: jobs:
build: build:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
name: checkout nosqlbench name: checkout nosqlbench
- uses: actions/setup-java@v3
name: setup java
with:
java-version: '17'
java-package: jdk
architecture: x64
distribution: 'temurin'
- uses: actions/setup-java@v3 - name: Cache Maven packages
name: setup java uses: actions/cache@v1
with: with:
java-version: '17' path: ~/.m2
java-package: jdk key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
architecture: x64 restore-keys: ${{ runner.os }}-m2
distribution: 'temurin'
- name: Cache Maven packages - name: mvn-package
uses: actions/cache@v1 run: mvn package
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
- name: mvn-package - name: export docs
run: mvn package run: nb5/target/nb5 export-docs
- name: mvn-verify - name: upload docs artifact
run: mvn verify uses: actions/upload-artifact@v3
with:
name: exported-docs
path: exported_docs.zip
- name: Capture - name: mvn verify
if: success() || failure() run: mvn verify
run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
- name: Archive Test Results - name: Capture
if: success() || failure() if: success() || failure()
uses: actions/upload-artifact@v3 run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
with:
name: test-results - name: Archive Test Results
path: logfiles.tar if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: test-results
path: logfiles.tar
docs:
needs: build
runs-on: ubuntu-20.04
steps:
- name: set git username
run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
- name: set git email
run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
- name: download exported-docs
uses: actions/download-artifact@v3
with:
name: exported-docs
- name: unzip docs
run: unzip exported_docs.zip
- run: ls -la
- name: clone nosqlbench-build-docs
env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
run: |
git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs
cd nosqlbench-build-docs
echo "files listing"
find .
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git
git remote -v
- name: push changes
env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
run: |
rsync -av --delete -I --exclude '_index.md' drivers/ nosqlbench-build-docs/site/content/docs/drivers
rsync -av --delete -I --exclude '_index.md' bindings/ nosqlbench-build-docs/site/content/docs/bindings
echo "previewdocs.nosqlbench.io" > nosqlbench-build-docs/site/staticCNAME
cd nosqlbench-build-docs
git add -A
CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
echo "found $CHANGES to push for doc updates"
if (( $CHANGES > 0 ))
then
git commit -m"docs update for $GITHUB_REF"
git push
fi
echo "push completed"

2
driver-pulsar/src/main/resources/pulsar.md → adapter-pulsar/src/main/resources/docs/pulsar.md
View File

@ -1,3 +1,5 @@
# Pulsar
- [1. Overview](#1-overview) - [1. Overview](#1-overview)
- [1.1. Issues Tracker](#11-issues-tracker) - [1.1. Issues Tracker](#11-issues-tracker)
- [2. NB Pulsar Driver Workload Definition Yaml File - High Level Structure](#2-nb-pulsar-driver-workload-definition-yaml-file---high-level-structure) - [2. NB Pulsar Driver Workload Definition Yaml File - High Level Structure](#2-nb-pulsar-driver-workload-definition-yaml-file---high-level-structure)

1
adapter-pulsar/src/main/resources/pulsar.md
View File

@ -1,3 +1,4 @@
# Table of contents
- [1. Overview](#1-overview) - [1. Overview](#1-overview)
- [1.1. Issues Tracker](#11-issues-tracker) - [1.1. Issues Tracker](#11-issues-tracker)
- [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload) - [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload)

1
adapter-s4j/src/main/resources/s4j.md
View File

@ -1,3 +1,4 @@
# S4J Adapter
- [1. Overview](#1-overview) - [1. Overview](#1-overview)
- [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload) - [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload)
- [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file) - [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file)

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java
View File

@ -19,23 +19,27 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.api.docsapi.BundledMarkdownManifest; import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
import io.nosqlbench.api.docsapi.Docs; import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder; import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.api.docsapi.DocsNameSpace;
import io.nosqlbench.nb.annotations.Maturity; import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.api.spi.SimpleServiceLoader; import io.nosqlbench.api.spi.SimpleServiceLoader;
import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set;
@Service(value = BundledMarkdownManifest.class, selector = "adapter-docs") @Service(value = BundledMarkdownManifest.class, selector = "drivers")
public class BundledDriverAdapterDocs implements BundledMarkdownManifest { public class BundledDriverAdapterDocs implements BundledMarkdownManifest {
@Override @Override
public DocsBinder getDocs() { public DocsBinder getDocs() {
Docs docs = new Docs().namespace("adapter-docs"); DocsBinder docs = new Docs();
SimpleServiceLoader<DriverAdapter> loader = new SimpleServiceLoader<>(DriverAdapter.class, Maturity.Any); SimpleServiceLoader<DriverAdapter> loader = new SimpleServiceLoader<>(DriverAdapter.class, Maturity.Any);
List<SimpleServiceLoader.Component<? extends DriverAdapter>> namedProviders = loader.getNamedProviders(); List<SimpleServiceLoader.Component<? extends DriverAdapter>> namedProviders = loader.getNamedProviders();
for (SimpleServiceLoader.Component<? extends DriverAdapter> namedProvider : namedProviders) { for (SimpleServiceLoader.Component<? extends DriverAdapter> namedProvider : namedProviders) {
DriverAdapter driverAdapter = namedProvider.provider.get(); DriverAdapter driverAdapter = namedProvider.provider.get();
DocsBinder bundledDocs = driverAdapter.getBundledDocs(); DocsBinder bundledDocs = driverAdapter.getBundledDocs();
docs.merge(bundledDocs); docs = docs.merge(bundledDocs);
} }
return docs; return docs;
} }

7
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java
View File

@ -28,11 +28,15 @@ import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO; import io.nosqlbench.api.content.NBIO;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.LongFunction; import java.util.function.LongFunction;
import java.util.stream.Stream;
/** /**
* <P>The DriverAdapter interface is expected to be the replacement * <P>The DriverAdapter interface is expected to be the replacement
@ -177,7 +181,7 @@ public interface DriverAdapter<OPTYPE extends Op, SPACETYPE> {
* @return A {@link DocsBinder} which describes docs to include for a given adapter. * @return A {@link DocsBinder} which describes docs to include for a given adapter.
*/ */
default DocsBinder getBundledDocs() { default DocsBinder getBundledDocs() {
Docs docs = new Docs().namespace("adapter-"+this.getAdapterName()); Docs docs = new Docs().namespace("drivers");
String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName(); String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName();
String cp_docspath = "docs/" + this.getAdapterName(); String cp_docspath = "docs/" + this.getAdapterName();
@ -185,6 +189,7 @@ public interface DriverAdapter<OPTYPE extends Op, SPACETYPE> {
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf); bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first(); Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first();
maindoc.map(Content::asPath).ifPresent(docs::addPath); maindoc.map(Content::asPath).ifPresent(docs::addPath);
return docs.asDocsBinder(); return docs.asDocsBinder();

85
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarAction.java
View File

@ -1,85 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar;
import com.codahale.metrics.Timer;
import io.nosqlbench.driver.pulsar.ops.PulsarOp;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
public class PulsarAction implements SyncAction {
private final static Logger logger = LogManager.getLogger(PulsarAction.class);
private final int slot;
private final PulsarActivity activity;
int maxTries = 1;
public PulsarAction(PulsarActivity activity, int slot) {
this.activity = activity;
this.slot = slot;
this.maxTries = activity.getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10);
}
@Override
public void init() {
}
@Override
public int runCycle(long cycle) {
// let's fail the action if some async operation failed
activity.failOnAsyncOperationFailure();
long start = System.nanoTime();
PulsarOp pulsarOp;
try (Timer.Context ctx = activity.getBindTimer().time()) {
LongFunction<? extends PulsarOp> readyPulsarOp = activity.getSequencer().apply(cycle);
pulsarOp = readyPulsarOp.apply(cycle);
} catch (Exception bindException) {
// if diagnostic mode ...
activity.getErrorHandler().handleError(bindException, cycle, 0);
throw new RuntimeException(
"while binding request in cycle " + cycle + ": " + bindException.getMessage(), bindException
);
}
for (int i = 0; i < maxTries; i++) {
Timer.Context ctx = activity.getExecuteTimer().time();
try {
// it is up to the pulsarOp to call Context#close when the activity is executed
// this allows us to track time for async operations
pulsarOp.run(ctx::close);
break;
} catch (RuntimeException err) {
ErrorDetail errorDetail = activity
.getErrorHandler()
.handleError(err, cycle, System.nanoTime() - start);
if (!errorDetail.isRetryable()) {
break;
}
}
}
return 0;
}
}

319
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java
View File

@ -1,319 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import io.nosqlbench.driver.pulsar.ops.PulsarOp;
import io.nosqlbench.driver.pulsar.ops.ReadyPulsarOp;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import io.nosqlbench.driver.pulsar.util.PulsarNBClientConf;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminBuilder;
import org.apache.pulsar.client.api.*;
import org.apache.pulsar.common.schema.KeyValueEncodingType;
import java.util.Map;
import java.util.Optional;
public class PulsarActivity extends SimpleActivity implements ActivityDefObserver {
private final static Logger logger = LogManager.getLogger(PulsarActivity.class);
private Counter bytesCounter;
private Histogram messageSizeHistogram;
private Timer bindTimer;
private Timer executeTimer;
private Timer createTransactionTimer;
private Timer commitTransactionTimer;
// Metrics for NB Pulsar driver milestone: https://github.com/nosqlbench/nosqlbench/milestone/11
// - end-to-end latency
private Histogram e2eMsgProcLatencyHistogram;
/**
* A histogram that tracks payload round-trip-time, based on a user-defined field in some sender
* system which can be interpreted as millisecond epoch time in the system's local time zone.
* This is paired with a field name of the same type to be extracted and reported in a meteric
* named 'payload-rtt'.
*/
private Histogram payloadRttHistogram;
// - message out of sequence error counter
private Counter msgErrOutOfSeqCounter;
// - message loss counter
private Counter msgErrLossCounter;
// - message duplicate (when dedup is enabled) error counter
private Counter msgErrDuplicateCounter;
private PulsarSpaceCache pulsarCache;
private PulsarNBClientConf pulsarNBClientConf;
private String pulsarSvcUrl;
private String webSvcUrl;
private PulsarAdmin pulsarAdmin;
private PulsarClient pulsarClient;
private Schema<?> pulsarSchema;
private NBErrorHandler errorHandler;
private OpSequence<OpDispenser<? extends PulsarOp>> sequencer;
private volatile Throwable asyncOperationFailure;
private boolean cycleratePerThread;
public PulsarActivity(ActivityDef activityDef) {
super(activityDef);
}
@Override
public void shutdownActivity() {
super.shutdownActivity();
if (pulsarCache == null) {
return;
}
for (PulsarSpace pulsarSpace : pulsarCache.getAssociatedPulsarSpace()) {
pulsarSpace.shutdownPulsarSpace();
}
}
@Override
public void initActivity() {
super.initActivity();
pulsarCache = new PulsarSpaceCache(this);
bytesCounter = ActivityMetrics.counter(activityDef, "bytes");
messageSizeHistogram = ActivityMetrics.histogram(activityDef, "message_size", this.getHdrDigits());
bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits());
executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits());
createTransactionTimer = ActivityMetrics.timer(activityDef, "create_transaction", this.getHdrDigits());
commitTransactionTimer = ActivityMetrics.timer(activityDef, "commit_transaction", this.getHdrDigits());
e2eMsgProcLatencyHistogram = ActivityMetrics.histogram(activityDef, "e2e_msg_latency", this.getHdrDigits());
payloadRttHistogram = ActivityMetrics.histogram(activityDef, "payload_rtt", this.getHdrDigits());
msgErrOutOfSeqCounter = ActivityMetrics.counter(activityDef, "err_msg_oos");
msgErrLossCounter = ActivityMetrics.counter(activityDef, "err_msg_loss");
msgErrDuplicateCounter = ActivityMetrics.counter(activityDef, "err_msg_dup");
String pulsarClntConfFile =
activityDef.getParams().getOptionalString("config").orElse("config.properties");
pulsarNBClientConf = new PulsarNBClientConf(pulsarClntConfFile);
pulsarSvcUrl =
activityDef.getParams().getOptionalString("service_url").orElse("pulsar://localhost:6650");
webSvcUrl =
activityDef.getParams().getOptionalString("web_url").orElse("http://localhost:8080");
initPulsarAdminAndClientObj();
createPulsarSchemaFromConf();
this.sequencer = createOpSequence((ot) -> new ReadyPulsarOp(ot, pulsarCache, this), false, Optional.empty());
setDefaultsFromOpSequence(sequencer);
onActivityDefUpdate(activityDef);
this.errorHandler = new NBErrorHandler(
() -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
this::getExceptionMetrics
);
cycleratePerThread = activityDef.getParams().takeBoolOrDefault("cyclerate_per_thread", false);
}
private final ThreadLocal<RateLimiter> cycleLimiterThreadLocal = ThreadLocal.withInitial(() -> {
if (super.getCycleLimiter() != null) {
return RateLimiters.createOrUpdate(this.getActivityDef(), "cycles", null,
super.getCycleLimiter().getRateSpec());
} else {
return null;
}
});
@Override
public RateLimiter getCycleLimiter() {
if (cycleratePerThread) {
return cycleLimiterThreadLocal.get();
} else {
return super.getCycleLimiter();
}
}
public NBErrorHandler getErrorHandler() { return errorHandler; }
public OpSequence<OpDispenser<? extends PulsarOp>> getSequencer() { return sequencer; }
public void failOnAsyncOperationFailure() {
if (asyncOperationFailure != null) {
throw new RuntimeException(asyncOperationFailure);
}
}
public void asyncOperationFailed(Throwable ex) {
this.asyncOperationFailure = ex;
}
/**
* Initialize
* - PulsarAdmin object for adding/deleting tenant, namespace, and topic
* - PulsarClient object for message publishing and consuming
*/
private void initPulsarAdminAndClientObj() {
PulsarAdminBuilder adminBuilder =
PulsarAdmin.builder()
.serviceHttpUrl(webSvcUrl);
ClientBuilder clientBuilder = PulsarClient.builder();
try {
Map<String, Object> clientConfMap = pulsarNBClientConf.getClientConfMap();
// Override "client.serviceUrl" setting in config.properties
clientConfMap.remove("serviceUrl");
clientBuilder.loadConf(clientConfMap).serviceUrl(pulsarSvcUrl);
// Pulsar Authentication
String authPluginClassName =
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.authPulginClassName.label);
String authParams =
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.authParams.label);
if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) {
adminBuilder.authentication(authPluginClassName, authParams);
clientBuilder.authentication(authPluginClassName, authParams);
}
String useTlsStr =
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.useTls.label);
boolean useTls = BooleanUtils.toBoolean(useTlsStr);
String tlsTrustCertsFilePath =
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label);
String tlsAllowInsecureConnectionStr =
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label);
boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr);
String tlsHostnameVerificationEnableStr =
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label);
boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr);
if ( useTls ) {
adminBuilder
.enableTlsHostnameVerification(tlsHostnameVerificationEnable);
clientBuilder
.enableTlsHostnameVerification(tlsHostnameVerificationEnable);
if (!StringUtils.isBlank(tlsTrustCertsFilePath)) {
adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath);
clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath);
}
}
// Put this outside "if (useTls)" block for easier handling of "tlsAllowInsecureConnection"
adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection);
clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection);
pulsarAdmin = adminBuilder.build();
pulsarClient = clientBuilder.build();
////////////////
// Not supported in Pulsar 2.8.0
//
// ClientConfigurationData configurationData = pulsarAdmin.getClientConfigData();
// logger.debug(configurationData.toString());
} catch (PulsarClientException e) {
logger.error("Fail to create PulsarAdmin and/or PulsarClient object from the global configuration!");
throw new RuntimeException("Fail to create PulsarAdmin and/or PulsarClient object from global configuration!");
}
}
/**
* Get Pulsar schema from the definition string
*/
private void createPulsarSchemaFromConf() {
pulsarSchema = buldSchemaFromDefinition("schema.type", "schema.definition");
// this is to allow KEY_VALUE schema
if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) {
Schema<?> pulsarKeySchema = buldSchemaFromDefinition("schema.key.type", "schema.key.definition");
Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype");
KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED;
if (encodingType != null) {
keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString());
}
pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType);
}
}
private Schema<?> buldSchemaFromDefinition(String schemaTypeConfEntry,
String schemaDefinitionConfEntry) {
Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry);
Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry);
String schemaType = (value != null) ? value.toString() : "";
Schema<?> result;
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType)) {
String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : "";
result = PulsarActivityUtil.getAvroSchema(schemaType, schemaDefStr);
} else if (PulsarActivityUtil.isPrimitiveSchemaTypeStr(schemaType)) {
result = PulsarActivityUtil.getPrimitiveTypeSchema(schemaType);
} else if (PulsarActivityUtil.isAutoConsumeSchemaTypeStr(schemaType)) {
result = Schema.AUTO_CONSUME();
} else {
throw new RuntimeException("Unsupported schema type string: " + schemaType + "; " +
"Only primitive type, Avro type and AUTO_CONSUME are supported at the moment!");
}
return result;
}
public PulsarNBClientConf getPulsarConf() { return this.pulsarNBClientConf;}
public String getPulsarSvcUrl() { return this.pulsarSvcUrl;}
public String getWebSvcUrl() { return this.webSvcUrl; }
public PulsarAdmin getPulsarAdmin() { return this.pulsarAdmin; }
public PulsarClient getPulsarClient() { return this.pulsarClient; }
public Schema<?> getPulsarSchema() { return pulsarSchema; }
public Counter getBytesCounter() { return bytesCounter; }
public Histogram getMessageSizeHistogram() { return messageSizeHistogram; }
public Timer getBindTimer() { return bindTimer; }
public Timer getExecuteTimer() { return this.executeTimer; }
public Timer getCreateTransactionTimer() { return createTransactionTimer; }
public Timer getCommitTransactionTimer() { return commitTransactionTimer; }
public Histogram getPayloadRttHistogram() {return payloadRttHistogram;}
public Histogram getE2eMsgProcLatencyHistogram() { return e2eMsgProcLatencyHistogram; }
public Counter getMsgErrOutOfSeqCounter() { return msgErrOutOfSeqCounter; }
public Counter getMsgErrLossCounter() { return msgErrLossCounter; }
public Counter getMsgErrDuplicateCounter() { return msgErrDuplicateCounter; }
}

52
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivityType.java
View File

@ -1,52 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar;
import io.nosqlbench.engine.api.activityapi.core.Action;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.annotations.Service;
@Service(value= ActivityType.class, selector="pulsar")
public class PulsarActivityType implements ActivityType<PulsarActivity> {
@Override
public ActionDispenser getActionDispenser(PulsarActivity activity) {
if (activity.getParams().getOptionalString("async").isPresent()) {
throw new RuntimeException("The async pulsar driver is not implemented yet.");
}
return new PulsarActionDispenser(activity);
}
@Override
public PulsarActivity getActivity(ActivityDef activityDef) {
return new PulsarActivity(activityDef);
}
private static class PulsarActionDispenser implements ActionDispenser {
private final PulsarActivity activity;
public PulsarActionDispenser(PulsarActivity activity) {
this.activity = activity;
}
@Override
public Action getAction(int slot) {
return new PulsarAction(activity, slot);
}
}
}

802
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpace.java
View File

@ -1,802 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Timer;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import io.nosqlbench.driver.pulsar.util.PulsarNBClientConf;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.Clusters;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.*;
import org.apache.pulsar.client.api.transaction.Transaction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
/**
* An instance of a pulsar client, along with all the cached objects which are normally
* associated with it during a client session in a typical application.
* A PulsarSpace is simply a named and cached set of objects which must be used together.
*/
public class PulsarSpace {
private final static Logger logger = LogManager.getLogger(PulsarSpace.class);
private final String spaceName;
private final ConcurrentHashMap<String, Producer<?>> producers = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Consumer<?>> consumers = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Reader<?>> readers = new ConcurrentHashMap<>();
private final PulsarActivity pulsarActivity;
private final ActivityDef activityDef;
private final PulsarNBClientConf pulsarNBClientConf;
private final String pulsarSvcUrl;
private final String webSvcUrl;
private final PulsarAdmin pulsarAdmin;
private final PulsarClient pulsarClient;
private final Schema<?> pulsarSchema;
private final Set<String> pulsarClusterMetadata = new HashSet<>();
private final Timer createTransactionTimer;
public PulsarSpace(String name, PulsarActivity pulsarActivity) {
this.spaceName = name;
this.pulsarActivity = pulsarActivity;
this.pulsarNBClientConf = pulsarActivity.getPulsarConf();
this.pulsarSvcUrl = pulsarActivity.getPulsarSvcUrl();
this.webSvcUrl = pulsarActivity.getWebSvcUrl();
this.pulsarAdmin = pulsarActivity.getPulsarAdmin();
this.pulsarClient = pulsarActivity.getPulsarClient();
this.pulsarSchema = pulsarActivity.getPulsarSchema();
this.activityDef = pulsarActivity.getActivityDef();
this.createTransactionTimer = pulsarActivity.getCreateTransactionTimer();
try {
Clusters clusters = pulsarAdmin.clusters();
List<String> stringList = clusters.getClusters();
CollectionUtils.addAll(pulsarClusterMetadata, stringList.listIterator());
} catch (PulsarAdminException e) {
// this is okay if you are connecting with a token that does not have access to the
// system configuration
logger.info("Could not get list of Pulsar Clusters from global configuration: " + e.getMessage());
}
}
public PulsarNBClientConf getPulsarClientConf() { return pulsarNBClientConf; }
public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; }
public PulsarClient getPulsarClient() { return pulsarClient; }
public Schema<?> getPulsarSchema() { return pulsarSchema; }
public String getPulsarSvcUrl() { return pulsarSvcUrl;}
public String getWebSvcUrl() { return webSvcUrl; }
public Set<String> getPulsarClusterMetadata() { return pulsarClusterMetadata; }
// Properly shut down all Pulsar objects (producers, consumers, etc.) that are associated with this space
public void shutdownPulsarSpace() {
try {
for (Producer<?> producer : producers.values()) {
if (producer != null) producer.close();
}
for (Consumer<?> consumer : consumers.values()) {
if (consumer != null) consumer.close();
}
for (Reader<?> reader : readers.values()) {
if (reader != null) reader.close();
}
if (pulsarAdmin != null) pulsarAdmin.close();
if (pulsarClient != null) pulsarClient.close();
}
catch (Exception e) {
throw new RuntimeException("Unexpected error when closing Pulsar objects!");
}
}
/**
* Get a proper Pulsar API metrics prefix depending on the API type
*
* @param apiType - Pulsar API type: producer, consumer, reader, etc.
* @param apiObjName - actual name of a producer, a consumer, a reader, etc.
* @param topicName - topic name
* @return String
*/
private String getPulsarAPIMetricsPrefix(String apiType, String apiObjName, String topicName) {
String apiMetricsPrefix;
if (!PulsarActivityUtil.isValidPulsarApiType(apiType)) {
throw new RuntimeException(
"Incorrect Pulsar API type. Valid type list: " + PulsarActivityUtil.getValidPulsarApiTypeList());
}
if (!StringUtils.isBlank(apiObjName)) {
apiMetricsPrefix = apiObjName + "_";
}
else {
// we want a meaningful name for the API object (producer, consumer, reader, etc.)
// we are not appending the topic name
apiMetricsPrefix = apiType;
if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label))
apiMetricsPrefix += producers.size();
else if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.CONSUMER.label))
apiMetricsPrefix += consumers.size();
else if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.READER.label))
apiMetricsPrefix += readers.size();
apiMetricsPrefix += "_";
}
apiMetricsPrefix += topicName + "_";
apiMetricsPrefix = apiMetricsPrefix
// default name for tests/demos (in all Pulsar examples) is persistent://public/default/test -> use just the topic name test
.replace("persistent://public/default/", "")
// always remove topic type
.replace("non-persistent://", "")
.replace("persistent://", "")
// persistent://tenant/namespace/topicname -> tenant_namespace_topicname
.replace("/","_");
return apiMetricsPrefix;
}
//////////////////////////////////////
// Producer Processing --> start
//////////////////////////////////////
//
private static class ProducerGaugeImpl implements Gauge<Object> {
private final Producer<?> producer;
private final Function<ProducerStats, Object> valueExtractor;
ProducerGaugeImpl(Producer<?> producer, Function<ProducerStats, Object> valueExtractor) {
this.producer = producer;
this.valueExtractor = valueExtractor;
}
@Override
public Object getValue() {
// see Pulsar bug https://github.com/apache/pulsar/issues/10100
// we need to synchronize on producer otherwise we could receive corrupted data
synchronized(producer) {
return valueExtractor.apply(producer.getStats());
}
}
}
static Gauge<Object> producerSafeExtractMetric(Producer<?> producer, Function<ProducerStats, Object> valueExtractor) {
return new ProducerGaugeImpl(producer, valueExtractor);
}
// Producer name is NOT mandatory
// - It can be set at either global level or cycle level
// - If set at both levels, cycle level setting takes precedence
private String getEffectiveProducerName(String cycleProducerName) {
if (!StringUtils.isBlank(cycleProducerName)) {
return cycleProducerName;
}
String globalProducerName = pulsarNBClientConf.getProducerName();
if (!StringUtils.isBlank(globalProducerName)) {
return globalProducerName;
}
return "";
}
public Supplier<Transaction> getTransactionSupplier() {
PulsarClient pulsarClient = getPulsarClient();
return () -> {
try (Timer.Context time = createTransactionTimer.time() ){
return pulsarClient
.newTransaction()
.build()
.get();
} catch (ExecutionException | InterruptedException err) {
if (logger.isWarnEnabled()) {
logger.warn("Error while starting a new transaction", err);
}
throw new RuntimeException(err);
} catch (PulsarClientException err) {
throw new RuntimeException("Transactions are not enabled on Pulsar Client, " +
"please set client.enableTransaction=true in your Pulsar Client configuration");
}
};
}
// Topic name IS mandatory
// - It must be set at either global level or cycle level
// - If set at both levels, cycle level setting takes precedence
private String getEffectiveProducerTopicName(String cycleTopicName) {
if (!StringUtils.isBlank(cycleTopicName)) {
return cycleTopicName;
}
String globalTopicName = pulsarNBClientConf.getProducerTopicName();
if (!StringUtils.isBlank(globalTopicName)) {
return globalTopicName;
}
throw new RuntimeException("Producer topic name must be set at either global level or cycle level!");
}
public Producer<?> getProducer(String cycleTopicName, String cycleProducerName) {
String topicName = getEffectiveProducerTopicName(cycleTopicName);
String producerName = getEffectiveProducerName(cycleProducerName);
if (StringUtils.isBlank(topicName)) {
throw new RuntimeException("Producer:: must specify a topic name");
}
String producerCacheKey = PulsarActivityUtil.buildCacheKey(producerName, topicName);
Producer<?> producer = producers.get(producerCacheKey);
if (producer == null) {
PulsarClient pulsarClient = getPulsarClient();
// Get other possible producer settings that are set at global level
Map<String, Object> producerConf = pulsarNBClientConf.getProducerConfMap();
// Remove global level settings: "topicName" and "producerName"
producerConf.remove(PulsarActivityUtil.PRODUCER_CONF_STD_KEY.topicName.label);
producerConf.remove(PulsarActivityUtil.PRODUCER_CONF_STD_KEY.producerName.label);
String producerMetricsPrefix = getPulsarAPIMetricsPrefix(
PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label,
producerName,
topicName);
try {
ProducerBuilder<?> producerBuilder = pulsarClient.
newProducer(pulsarSchema).
loadConf(producerConf).
topic(topicName);
if (!StringUtils.isAnyBlank(producerName)) {
producerBuilder = producerBuilder.producerName(producerName);
}
producer = producerBuilder.create();
producers.put(producerCacheKey, producer);
ActivityMetrics.gauge(activityDef,
producerMetricsPrefix + "total_bytes_sent",
producerSafeExtractMetric(producer, (s -> s.getTotalBytesSent() + s.getNumBytesSent())));
ActivityMetrics.gauge(activityDef,
producerMetricsPrefix + "total_msg_sent",
producerSafeExtractMetric(producer, (s -> s.getTotalMsgsSent() + s.getNumMsgsSent())));
ActivityMetrics.gauge(activityDef,
producerMetricsPrefix + "total_send_failed",
producerSafeExtractMetric(producer, (s -> s.getTotalSendFailed() + s.getNumSendFailed())));
ActivityMetrics.gauge(activityDef,
producerMetricsPrefix + "total_ack_received",
producerSafeExtractMetric(producer,(s -> s.getTotalAcksReceived() + s.getNumAcksReceived())));
ActivityMetrics.gauge(activityDef,
producerMetricsPrefix + "send_bytes_rate",
producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate));
ActivityMetrics.gauge(activityDef,
producerMetricsPrefix + "send_msg_rate",
producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate));
}
catch (PulsarClientException ple) {
throw new RuntimeException("Unable to create a Pulsar producer!", ple);
}
}
return producer;
}
//
//////////////////////////////////////
// Producer Processing <-- end
//////////////////////////////////////
//////////////////////////////////////
// Consumer Processing --> start
//////////////////////////////////////
//
private static class ConsumerGaugeImpl implements Gauge<Object> {
private final Consumer<?> consumer;
private final Function<ConsumerStats, Object> valueExtractor;
ConsumerGaugeImpl(Consumer<?> consumer, Function<ConsumerStats, Object> valueExtractor) {
this.consumer = consumer;
this.valueExtractor = valueExtractor;
}
@Override
public Object getValue() {
// see Pulsar bug https://github.com/apache/pulsar/issues/10100
// - this is a bug report for producer stats.
// - assume this also applies to consumer stats.
synchronized(consumer) {
return valueExtractor.apply(consumer.getStats());
}
}
}
static Gauge<Object> consumerSafeExtractMetric(Consumer<?> consumer, Function<ConsumerStats, Object> valueExtractor) {
return new ConsumerGaugeImpl(consumer, valueExtractor);
}
private String getEffectiveSubscriptionName(String cycleSubscriptionName) {
if (!StringUtils.isBlank(cycleSubscriptionName)) {
return cycleSubscriptionName;
}
String globalSubscriptionName = pulsarNBClientConf.getConsumerSubscriptionName();
if (!StringUtils.isBlank(globalSubscriptionName)) {
return globalSubscriptionName;
}
throw new RuntimeException("Consumer::Subscription name must be set at either global level or cycle level!");
}
private String getEffectiveSubscriptionTypeStr(String cycleSubscriptionType) {
if (!StringUtils.isBlank(cycleSubscriptionType)) {
return cycleSubscriptionType;
}
String globalSubscriptionType = pulsarNBClientConf.getConsumerSubscriptionType();
if (!StringUtils.isBlank(globalSubscriptionType)) {
return globalSubscriptionType;
}
return "";
}
private SubscriptionType getEffectiveSubscriptionType(String cycleSubscriptionType) {
String effectiveSubscriptionStr = getEffectiveSubscriptionTypeStr(cycleSubscriptionType);
SubscriptionType subscriptionType = SubscriptionType.Exclusive;
if (!StringUtils.isBlank(effectiveSubscriptionStr)) {
if (!PulsarActivityUtil.isValidSubscriptionType(effectiveSubscriptionStr)) {
throw new RuntimeException("Consumer::Invalid subscription type (\"" +
effectiveSubscriptionStr + "\"). \nValid subscription types: " + PulsarActivityUtil.getValidSubscriptionTypeList());
} else {
subscriptionType = SubscriptionType.valueOf(effectiveSubscriptionStr);
}
}
return subscriptionType;
}
private String getEffectiveConsumerName(String cycleConsumerName) {
if (!StringUtils.isBlank(cycleConsumerName)) {
return cycleConsumerName;
}
String globalConsumerName = pulsarNBClientConf.getConsumerName();
if (!StringUtils.isBlank(globalConsumerName)) {
return globalConsumerName;
}
return "";
}
public Consumer<?> getConsumer(String cycleTopicName,
String cycleSubscriptionName,
String cycleSubscriptionType,
String cycleConsumerName,
String cycleKeySharedSubscriptionRanges) {
String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName);
SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType);
String consumerName = getEffectiveConsumerName(cycleConsumerName);
if (StringUtils.isAnyBlank(cycleTopicName, subscriptionName)) {
throw new RuntimeException("Consumer:: must specify a topic name and a subscription name");
}
String consumerCacheKey = PulsarActivityUtil.buildCacheKey(consumerName, subscriptionName, cycleTopicName);
Consumer<?> consumer = consumers.get(consumerCacheKey);
if (consumer == null) {
PulsarClient pulsarClient = getPulsarClient();
// Get other possible consumer settings that are set at global level
Map<String, Object> consumerConf = new HashMap<>(pulsarNBClientConf.getConsumerConfMap());
// Remove global level settings:
// - "topicNames", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName"
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
// Remove non-standard consumer configuration properties
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
try {
ConsumerBuilder<?> consumerBuilder = pulsarClient.
newConsumer(pulsarSchema).
loadConf(consumerConf).
topic(cycleTopicName).
subscriptionName(subscriptionName).
subscriptionType(subscriptionType);
if (subscriptionType == SubscriptionType.Key_Shared) {
KeySharedPolicy keySharedPolicy = KeySharedPolicy.autoSplitHashRange();
if (cycleKeySharedSubscriptionRanges != null && !cycleKeySharedSubscriptionRanges.isEmpty()) {
Range[] ranges = parseRanges(cycleKeySharedSubscriptionRanges);
logger.info("Configuring KeySharedPolicy#stickyHashRange with ranges {}", ranges);
keySharedPolicy = KeySharedPolicy.stickyHashRange().ranges(ranges);
}
consumerBuilder.keySharedPolicy(keySharedPolicy);
}
if (!StringUtils.isBlank(consumerName)) {
consumerBuilder = consumerBuilder.consumerName(consumerName);
}
consumer = consumerBuilder.subscribe();
String consumerMetricsPrefix = getPulsarAPIMetricsPrefix(
PulsarActivityUtil.PULSAR_API_TYPE.CONSUMER.label,
consumerName,
cycleTopicName);
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "total_bytes_recv",
consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "total_msg_recv",
consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "total_recv_failed",
consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "total_acks_sent",
consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "recv_bytes_rate",
consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "recv_msg_rate",
consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived));
} catch (PulsarClientException ple) {
ple.printStackTrace();
throw new RuntimeException("Unable to create a Pulsar consumer!");
}
consumers.put(consumerCacheKey, consumer);
}
return consumer;
}
private static Range[] parseRanges(String ranges) {
if (ranges == null || ranges.isEmpty()) {
return new Range[0];
}
String[] split = ranges.split(",");
Range[] result = new Range[split.length];
for (int i = 0; i < split.length; i++) {
String range = split[i];
int pos = range.indexOf("..");
if (pos <= 0) {
throw new IllegalArgumentException("Invalid range '" + range + "'");
}
try {
int start = Integer.parseInt(range.substring(0, pos));
int end = Integer.parseInt(range.substring(pos + 2));
result[i] = Range.of(start, end);
} catch (NumberFormatException err) {
throw new IllegalArgumentException("Invalid range '" + range + "'");
}
}
return result;
}
//
//////////////////////////////////////
// Consumer Processing <-- end
//////////////////////////////////////
//////////////////////////////////////
// Multi-topic Consumer Processing --> start
//////////////////////////////////////
//
private String getEffectiveConsumerTopicNameListStr(String cycleTopicNames) {
if (!StringUtils.isBlank(cycleTopicNames)) {
return cycleTopicNames;
}
String globalTopicNames = pulsarNBClientConf.getConsumerTopicNames();
if (!StringUtils.isBlank(globalTopicNames)) {
return globalTopicNames;
}
return "";
}
private List<String> getEffectiveConsumerTopicNameList(String cycleTopicNames) {
String effectiveTopicNamesStr = getEffectiveConsumerTopicNameListStr(cycleTopicNames);
String[] names = effectiveTopicNamesStr.split("[;,]");
ArrayList<String> effectiveTopicNameList = new ArrayList<>();
for (String name : names) {
if (!StringUtils.isBlank(name))
effectiveTopicNameList.add(name.trim());
}
return effectiveTopicNameList;
}
private String getEffectiveConsumerTopicPatternStr(String cycleTopicsPattern) {
if (!StringUtils.isBlank(cycleTopicsPattern)) {
return cycleTopicsPattern;
}
String globalTopicsPattern = pulsarNBClientConf.getConsumerTopicPattern();
if (!StringUtils.isBlank(globalTopicsPattern)) {
return globalTopicsPattern;
}
return "";
}
private Pattern getEffectiveConsumerTopicPattern(String cycleTopicsPattern) {
String effectiveTopicsPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicsPattern);
Pattern topicsPattern;
try {
if (!StringUtils.isBlank(effectiveTopicsPatternStr))
topicsPattern = Pattern.compile(effectiveTopicsPatternStr);
else
topicsPattern = null;
} catch (PatternSyntaxException pse) {
topicsPattern = null;
}
return topicsPattern;
}
public Consumer<?> getMultiTopicConsumer(
String cycleTopicUri,
String cycleTopicNameList,
String cycleTopicsPattern,
String cycleSubscriptionName,
String cycleSubscriptionType,
String cycleConsumerName) {
List<String> topicNameList = getEffectiveConsumerTopicNameList(cycleTopicNameList);
String topicsPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicsPattern);
Pattern topicsPattern = getEffectiveConsumerTopicPattern(cycleTopicsPattern);
String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName);
SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType);
String consumerName = getEffectiveConsumerName(cycleConsumerName);
if ( subscriptionType.equals(SubscriptionType.Exclusive) && (activityDef.getThreads() > 1) ) {
throw new RuntimeException("Consumer:: trying to create multiple consumers of " +
"\"Exclusive\" subscription type under the same subscription name to the same topic!");
}
if (StringUtils.isBlank(cycleTopicUri) && topicNameList.isEmpty() && (topicsPattern == null)) {
throw new RuntimeException("Consumer:: \"topic_uri\", \"topic_names\" and \"topics_pattern\" parameters can't be all empty/invalid!");
}
// precedence sequence:
// topic_names (consumer statement param) >
// topics_pattern (consumer statement param) >
// topic_uri (document level param)
String consumerTopicListString;
if (!topicNameList.isEmpty()) {
consumerTopicListString = String.join("|", topicNameList);
} else if (topicsPattern != null) {
consumerTopicListString = topicsPatternStr;
} else {
consumerTopicListString = cycleTopicUri;
}
String consumerCacheKey = PulsarActivityUtil.buildCacheKey(
consumerName,
subscriptionName,
consumerTopicListString);
Consumer<?> consumer = consumers.get(consumerCacheKey);
if (consumer == null) {
PulsarClient pulsarClient = getPulsarClient();
// Get other possible producer settings that are set at global level
Map<String, Object> consumerConf = new HashMap<>(pulsarNBClientConf.getConsumerConfMap());
// Remove global level settings:
// - "topicNameList", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName"
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
// Remove non-standard consumer configuration properties
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
try {
ConsumerBuilder<?> consumerBuilder = pulsarClient.newConsumer(pulsarSchema).
loadConf(consumerConf).
subscriptionName(subscriptionName).
subscriptionType(subscriptionType).
consumerName(consumerName);
if (!topicNameList.isEmpty()) {
consumerBuilder = consumerBuilder.topics(topicNameList);
} else if (topicsPattern != null) {
consumerBuilder = consumerBuilder.topicsPattern(topicsPattern);
} else {
consumerBuilder = consumerBuilder.topic(cycleTopicUri);
}
consumer = consumerBuilder.subscribe();
String consumerMetricsPrefix = getPulsarAPIMetricsPrefix(
PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label,
consumerName,
consumerTopicListString);
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "totalBytesRecvd",
consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "totalMsgsRecvd",
consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "totalRecvdFailed",
consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "totalAcksSent",
consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent())));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "recvdBytesRate",
consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived));
ActivityMetrics.gauge(activityDef,
consumerMetricsPrefix + "recvdMsgsRate",
consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived));
} catch (PulsarClientException ple) {
ple.printStackTrace();
throw new RuntimeException("Unable to create a Pulsar consumer!");
}
consumers.put(consumerCacheKey, consumer);
}
return consumer;
}
//
//////////////////////////////////////
// Multi-topic Consumer Processing <-- end
//////////////////////////////////////
//////////////////////////////////////
// Reader Processing --> Start
//////////////////////////////////////
private String getEffectiveReaderTopicName(String cycleReaderTopicName) {
if (!StringUtils.isBlank(cycleReaderTopicName)) {
return cycleReaderTopicName;
}
String globalReaderTopicName = pulsarNBClientConf.getReaderTopicName();
if (!StringUtils.isBlank(globalReaderTopicName)) {
return globalReaderTopicName;
}
throw new RuntimeException("Reader:: Reader topic name must be set at either global level or cycle level!");
}
private String getEffectiveReaderName(String cycleReaderName) {
if (!StringUtils.isBlank(cycleReaderName)) {
return cycleReaderName;
}
String globalReaderName = pulsarNBClientConf.getConsumerName();
if (!StringUtils.isBlank(globalReaderName)) {
return globalReaderName;
}
return "";
}
private String getEffectiveStartMsgPosStr(String cycleStartMsgPosStr) {
if (!StringUtils.isBlank(cycleStartMsgPosStr)) {
return cycleStartMsgPosStr;
}
String globalStartMsgPosStr = pulsarNBClientConf.getStartMsgPosStr();
if (!StringUtils.isBlank(globalStartMsgPosStr)) {
return globalStartMsgPosStr;
}
return PulsarActivityUtil.READER_MSG_POSITION_TYPE.latest.label;
}
public Reader<?> getReader(String cycleTopicName,
String cycleReaderName,
String cycleStartMsgPos) {
String topicName = getEffectiveReaderTopicName(cycleTopicName);
String readerName = getEffectiveReaderName(cycleReaderName);
String startMsgPosStr = getEffectiveStartMsgPosStr(cycleStartMsgPos);
if (!PulsarActivityUtil.isValideReaderStartPosition(startMsgPosStr)) {
throw new RuntimeException("Reader:: Invalid value for reader start message position!");
}
String readerCacheKey = PulsarActivityUtil.buildCacheKey(topicName, readerName, startMsgPosStr);
Reader<?> reader = readers.get(readerCacheKey);
if (reader == null) {
PulsarClient pulsarClient = getPulsarClient();
Map<String, Object> readerConf = pulsarNBClientConf.getReaderConfMap();
// Remove global level settings: "topicName" and "readerName"
readerConf.remove(PulsarActivityUtil.READER_CONF_STD_KEY.topicName.label);
readerConf.remove(PulsarActivityUtil.READER_CONF_STD_KEY.readerName.label);
// Remove non-standard reader configuration properties
readerConf.remove(PulsarActivityUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label);
try {
ReaderBuilder<?> readerBuilder = pulsarClient.
newReader(pulsarSchema).
loadConf(readerConf).
topic(topicName).
readerName(readerName);
MessageId startMsgId = MessageId.latest;
if (startMsgPosStr.equalsIgnoreCase(PulsarActivityUtil.READER_MSG_POSITION_TYPE.earliest.label)) {
startMsgId = MessageId.earliest;
}
//TODO: custom start message position is NOT supported yet
//else if (startMsgPosStr.startsWith(PulsarActivityUtil.READER_MSG_POSITION_TYPE.custom.label)) {
// startMsgId = MessageId.latest;
//}
reader = readerBuilder.startMessageId(startMsgId).create();
} catch (PulsarClientException ple) {
ple.printStackTrace();
throw new RuntimeException("Unable to create a Pulsar reader!");
}
readers.put(readerCacheKey, reader);
}
return reader;
}
//////////////////////////////////////
// Reader Processing <-- end
//////////////////////////////////////
}

52
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpaceCache.java
View File

@ -1,52 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar;
import java.util.concurrent.ConcurrentHashMap;
/**
* To enable flexibility in testing methods, each object graph which is used within
* the pulsar API is kept within a single umbrella called the PulsarSpace.
* This allows for clients, producers, and consumers to remain connected and
* cached in a useful way.
*/
public class PulsarSpaceCache {
// TODO: Implement cache limits
// TODO: Implement variant cache eviction behaviors (halt, warn, LRU)
private final PulsarActivity activity;
private final ConcurrentHashMap<String, PulsarSpace> clientScopes = new ConcurrentHashMap<>();
public PulsarSpaceCache(PulsarActivity pulsarActivity) {
this.activity = pulsarActivity;
}
public Iterable<PulsarSpace> getAssociatedPulsarSpace() {
return clientScopes.values();
}
public PulsarActivity getAssociatedPulsarActivity() {
return activity;
}
public PulsarSpace getPulsarSpace(String name) {
return clientScopes.computeIfAbsent(name, spaceName -> new PulsarSpace(spaceName, activity));
}
public PulsarActivity getActivity() { return activity; }
}

24
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverParamException.java
View File

@ -1,24 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.exception;
public class PulsarDriverParamException extends RuntimeException {
public PulsarDriverParamException(String message) {
super(message);
}
}

25
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnexpectedException.java
View File

@ -1,25 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.exception;
public class PulsarDriverUnexpectedException extends RuntimeException {
public PulsarDriverUnexpectedException(String message) {
super(message);
}
public PulsarDriverUnexpectedException(Exception e) { super(e); }
}

23
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnsupportedOpException.java
View File

@ -1,23 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.exception;
public class PulsarDriverUnsupportedOpException extends RuntimeException {
public PulsarDriverUnsupportedOpException() { super("Unsupported Pulsar driver operation type"); }
}

24
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/EndToEndStartingTimeSource.java
View File

@ -1,24 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
public enum EndToEndStartingTimeSource {
NONE, // no end-to-end latency calculation
MESSAGE_PUBLISH_TIME, // use message publish timestamp
MESSAGE_EVENT_TIME, // use message event timestamp
MESSAGE_PROPERTY_E2E_STARTING_TIME // use message property called "e2e_starting_time" as the timestamp
}

103
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandler.java
View File

@ -1,103 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import java.util.*;
import org.apache.commons.lang3.RandomUtils;
/**
* Handles adding a monotonic sequence number to message properties of sent messages
*/
class MessageSequenceNumberSendingHandler {
static final int SIMULATED_ERROR_PROBABILITY_PERCENTAGE = 10;
long number = 1;
Queue<Long> outOfOrderNumbers;
public long getNextSequenceNumber(Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> simulatedErrorTypes) {
return getNextSequenceNumber(simulatedErrorTypes, SIMULATED_ERROR_PROBABILITY_PERCENTAGE);
}
long getNextSequenceNumber(Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> simulatedErrorTypes, int errorProbabilityPercentage) {
simulateError(simulatedErrorTypes, errorProbabilityPercentage);
return nextNumber();
}
private void simulateError(Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> simulatedErrorTypes, int errorProbabilityPercentage) {
if (!simulatedErrorTypes.isEmpty() && shouldSimulateError(errorProbabilityPercentage)) {
int selectIndex = 0;
int numberOfErrorTypes = simulatedErrorTypes.size();
if (numberOfErrorTypes > 1) {
// pick one of the simulated error type randomly
selectIndex = RandomUtils.nextInt(0, numberOfErrorTypes);
}
PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE errorType = simulatedErrorTypes.stream()
.skip(selectIndex)
.findFirst()
.get();
switch (errorType) {
case OutOfOrder:
// simulate message out of order
injectMessagesOutOfOrder();
break;
case MsgDup:
// simulate message duplication
injectMessageDuplication();
break;
case MsgLoss:
// simulate message loss
injectMessageLoss();
break;
}
}
}
private boolean shouldSimulateError(int errorProbabilityPercentage) {
// Simulate error with the specified probability
return RandomUtils.nextInt(0, 100) < errorProbabilityPercentage;
}
long nextNumber() {
if (outOfOrderNumbers != null) {
long nextNumber = outOfOrderNumbers.poll();
if (outOfOrderNumbers.isEmpty()) {
outOfOrderNumbers = null;
}
return nextNumber;
}
return number++;
}
void injectMessagesOutOfOrder() {
if (outOfOrderNumbers == null) {
outOfOrderNumbers = new ArrayDeque<>(Arrays.asList(number + 2, number, number + 1));
number += 3;
}
}
void injectMessageDuplication() {
if (outOfOrderNumbers == null) {
number--;
}
}
void injectMessageLoss() {
if (outOfOrderNumbers == null) {
number++;
}
}
}

46
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminMapper.java
View File

@ -1,46 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import java.util.function.LongFunction;
/**
* This maps a set of specifier functions to a pulsar operation. The result contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public abstract class PulsarAdminMapper extends PulsarOpMapper {
protected final LongFunction<Boolean> adminDelOpFunc;
protected PulsarAdminMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc) {
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
this.adminDelOpFunc = adminDelOpFunc;
}
}

62
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceMapper.java
View File

@ -1,62 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import java.util.Set;
import java.util.function.LongFunction;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class PulsarAdminNamespaceMapper extends PulsarAdminMapper {
private final LongFunction<String> namespaceFunc;
public PulsarAdminNamespaceMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc,
LongFunction<String> namespaceFunc)
{
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc);
this.namespaceFunc = namespaceFunc;
}
@Override
public PulsarOp apply(long value) {
boolean asyncApi = asyncApiFunc.apply(value);
boolean adminDelOp = adminDelOpFunc.apply(value);
String namespace = namespaceFunc.apply(value);
return new PulsarAdminNamespaceOp(
clientSpace,
asyncApi,
adminDelOp,
namespace);
}
}

101
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceOp.java
View File

@ -1,101 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.Namespaces;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import java.util.concurrent.CompletableFuture;
public class PulsarAdminNamespaceOp extends PulsarAdminOp {
private final static Logger logger = LogManager.getLogger(PulsarAdminNamespaceOp.class);
private final String fullNsName;
public PulsarAdminNamespaceOp(PulsarSpace clientSpace,
boolean asyncApi,
boolean adminDelOp,
String fullNsName)
{
super(clientSpace, asyncApi, adminDelOp);
this.fullNsName = fullNsName;
}
@Override
public void run() {
// Do nothing if the namespace name is empty
if ( StringUtils.isBlank(fullNsName) ) return;
PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin();
Namespaces namespaces = pulsarAdmin.namespaces();
// Admin API - create tenants and namespaces
if (!adminDelOp) {
try {
if (!asyncApi) {
namespaces.createNamespace(fullNsName);
logger.trace("Successfully created namespace \"" + fullNsName + "\" synchronously!");
} else {
CompletableFuture<Void> future = namespaces.createNamespaceAsync(fullNsName);
future.whenComplete((unused, throwable) ->
logger.trace("Successfully created namespace \"" + fullNsName + "\" asynchronously!"))
.exceptionally(ex -> {
logger.error("Failed to create namespace \"" + fullNsName + "\" asynchronously!:" + ex.getMessage());
return null;
});
}
}
catch (PulsarAdminException.ConflictException ce) {
// do nothing if the namespace already exists
}
catch (PulsarAdminException e) {
e.printStackTrace();
throw new RuntimeException("Unexpected error when creating pulsar namespace: " + fullNsName);
}
}
// Admin API - delete tenants and namespaces
else {
try {
if (!asyncApi) {
namespaces.deleteNamespace(fullNsName, true);
logger.trace("Successfully deleted namespace \"" + fullNsName + "\" synchronously!");
} else {
CompletableFuture<Void> future = namespaces.deleteNamespaceAsync(fullNsName, true);
future.whenComplete((unused, throwable) ->
logger.trace("Successfully deleted namespace \"" + fullNsName + "\" asynchronously!"))
.exceptionally(ex -> {
logger.error("Failed to delete namespace \"" + fullNsName + "\" asynchronously!");
return null;
});
}
}
catch (PulsarAdminException.NotFoundException nfe) {
// do nothing if the namespace doesn't exist
}
catch (PulsarAdminException e) {
e.printStackTrace();
throw new RuntimeException("Unexpected error when deleting pulsar namespace: " + fullNsName);
}
}
}
}

39
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminOp.java
View File

@ -1,39 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public abstract class PulsarAdminOp extends SyncPulsarOp {
private final static Logger logger = LogManager.getLogger(PulsarAdminOp.class);
protected final PulsarSpace clientSpace;
protected final boolean asyncApi;
protected final boolean adminDelOp;
protected PulsarAdminOp(PulsarSpace clientSpace,
boolean asyncApi,
boolean adminDelOp)
{
this.clientSpace = clientSpace;
this.asyncApi = asyncApi;
this.adminDelOp = adminDelOp;
}
}

72
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantMapper.java
View File

@ -1,72 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import java.util.Set;
import java.util.function.LongFunction;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class PulsarAdminTenantMapper extends PulsarAdminMapper {
private final LongFunction<Set<String>> adminRolesFunc;
private final LongFunction<Set<String>> allowedClustersFunc;
private final LongFunction<String> tenantFunc;
public PulsarAdminTenantMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc,
LongFunction<Set<String>> adminRolesFunc,
LongFunction<Set<String>> allowedClustersFunc,
LongFunction<String> tenantFunc)
{
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc);
this.adminRolesFunc = adminRolesFunc;
this.allowedClustersFunc = allowedClustersFunc;
this.tenantFunc = tenantFunc;
}
@Override
public PulsarOp apply(long value) {
boolean asyncApi = asyncApiFunc.apply(value);
boolean adminDelOp = adminDelOpFunc.apply(value);
Set<String> adminRoleSet = adminRolesFunc.apply(value);
Set<String> allowedClusterSet = allowedClustersFunc.apply(value);
String tenant = tenantFunc.apply(value);
return new PulsarAdminTenantOp(
clientSpace,
asyncApi,
adminDelOp,
adminRoleSet,
allowedClusterSet,
tenant);
}
}

128
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantOp.java
View File

@ -1,128 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.*;
import org.apache.pulsar.common.policies.data.TenantInfo;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
public class PulsarAdminTenantOp extends PulsarAdminOp {
private final static Logger logger = LogManager.getLogger(PulsarAdminTenantOp.class);
private final Set<String> adminRoleSet;
private final Set<String> allowedClusterSet;
private final String tenant;
public PulsarAdminTenantOp(PulsarSpace clientSpace,
boolean asyncApi,
boolean adminDelOp,
Set<String> adminRoleSet,
Set<String> allowedClusterSet,
String tenant)
{
super(clientSpace, asyncApi, adminDelOp);
this.adminRoleSet = adminRoleSet;
this.allowedClusterSet = allowedClusterSet;
this.tenant = tenant;
}
@Override
public void run() {
// Do nothing if the tenant name is empty
if ( StringUtils.isBlank(tenant) ) return;
PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin();
Tenants tenants = pulsarAdmin.tenants();
Namespaces namespaces = pulsarAdmin.namespaces();
// Admin API - create tenants and namespaces
if (!adminDelOp) {
TenantInfo tenantInfo = TenantInfo.builder()
.adminRoles(adminRoleSet)
.allowedClusters(!allowedClusterSet.isEmpty() ? allowedClusterSet : clientSpace.getPulsarClusterMetadata())
.build();
try {
if (!asyncApi) {
tenants.createTenant(tenant, tenantInfo);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync creation of tenant {}", tenant);
}
} else {
CompletableFuture<Void> future = tenants.createTenantAsync(tenant, tenantInfo);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug("Successful async creation of tenant {}", tenant);
}
}).exceptionally(ex -> {
logger.error("Failed async creation of tenant {}", tenant);
return null;
});
}
}
catch (PulsarAdminException.ConflictException ce) {
// do nothing if the tenant already exists
}
catch (PulsarAdminException e) {
e.printStackTrace();
throw new RuntimeException("Unexpected error when creating pulsar tenant: " + tenant);
}
}
// Admin API - delete tenants and namespaces
else {
try {
int nsNum = namespaces.getNamespaces(tenant).size();
// Only delete a tenant when there is no underlying namespaces
if ( nsNum == 0 ) {
if (!asyncApi) {
tenants.deleteTenant(tenant);
if (logger.isDebugEnabled()) {
logger.debug("Successful sync deletion of tenant {}", tenant);
}
} else {
CompletableFuture<Void> future = tenants.deleteTenantAsync(tenant);
future.whenComplete((unused, throwable) -> {
if (logger.isDebugEnabled()) {
logger.debug("Successful async deletion of tenant {}", tenant);
}
}).exceptionally(ex -> {
if (logger.isDebugEnabled()) {
logger.error("Failed async deletion of tenant {}", tenant);
}
return null;
});
}
}
}
catch (PulsarAdminException.NotFoundException nfe) {
// do nothing if the tenant doesn't exist
}
catch (PulsarAdminException e) {
e.printStackTrace();
throw new RuntimeException("Unexpected error when deleting pulsar tenant: " + tenant);
}
}
}
}

91
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicMapper.java
View File

@ -1,91 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.function.LongFunction;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class PulsarAdminTopicMapper extends PulsarAdminMapper {
private final LongFunction<String> topicUriFunc;
private final LongFunction<String> enablePartionFunc;
private final LongFunction<String> partitionNumFunc;
public PulsarAdminTopicMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc,
LongFunction<String> topicUriFunc,
LongFunction<String> enablePartionFunc,
LongFunction<String> partitionNumFunc)
{
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc);
this.topicUriFunc = topicUriFunc;
this.enablePartionFunc = enablePartionFunc;
this.partitionNumFunc = partitionNumFunc;
}
@Override
public PulsarOp apply(long value) {
String topicUri = topicUriFunc.apply(value);
String enablePartitionStr = enablePartionFunc.apply(value);
String partitionNumStr = partitionNumFunc.apply(value);
boolean asyncApi = asyncApiFunc.apply(value);
boolean adminDelOp = adminDelOpFunc.apply(value);
if ( StringUtils.isBlank(topicUri) ) {
throw new RuntimeException("\"topic_uri\" parameter can't be empty when creating a Pulsar topic!");
}
boolean partitionTopic = BooleanUtils.toBoolean(enablePartitionStr);
boolean invalidPartStr;
int partitionNum = 0;
if ( StringUtils.isBlank(partitionNumStr) || !StringUtils.isNumeric(partitionNumStr) ) {
invalidPartStr = true;
} else {
partitionNum = Integer.parseInt(partitionNumStr);
invalidPartStr = (partitionNum <= 0);
}
if (partitionTopic && invalidPartStr) {
throw new RuntimeException("Invalid specified value for \"partition_num\" parameter when creating partitioned topic!");
}
return new PulsarAdminTopicOp(
clientSpace,
topicUri,
partitionTopic,
partitionNum,
asyncApi,
adminDelOp);
}
}

159
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicOp.java
View File

@ -1,159 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.admin.Topics;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
public class PulsarAdminTopicOp extends PulsarAdminOp {
private final static Logger logger = LogManager.getLogger(PulsarAdminTopicOp.class);
private final String topicUri;
private final boolean partitionTopic;
private final int partitionNum;
private final String fullNsName;
public PulsarAdminTopicOp(PulsarSpace clientSpace,
String topicUri,
boolean partitionTopic,
int partitionNum,
boolean asyncApi,
boolean adminDelOp)
{
super(clientSpace, asyncApi, adminDelOp);
this.topicUri = topicUri;
this.partitionTopic = partitionTopic;
this.partitionNum = partitionNum;
this.fullNsName = PulsarActivityUtil.getFullNamespaceName(this.topicUri);
}
// Check whether the specified topic already exists
private boolean checkTopicExistence(Topics topics, String topicUri) {
// Check the existence of the topic
List<String> topicListWorkingArea = new ArrayList<>();
try {
if (!partitionTopic) {
topicListWorkingArea = topics.getList(fullNsName);
}
else {
topicListWorkingArea = topics.getPartitionedTopicList(fullNsName);
}
}
catch (PulsarAdminException.NotFoundException nfe) {
// do nothing
}
catch (PulsarAdminException e) {
e.printStackTrace();
throw new RuntimeException("Failed to retrieve topic info.for pulsar namespace: " + fullNsName);
}
return ( !topicListWorkingArea.isEmpty() && topicListWorkingArea.contains(topicUri) );
}
@Override
public void run() {
PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin();
Topics topics = pulsarAdmin.topics();
try {
// Create the topic
if (!adminDelOp) {
if (!partitionTopic) {
if (!asyncApi) {
topics.createNonPartitionedTopic(topicUri);
logger.trace("Successfully created non-partitioned topic \"" + topicUri + "\" synchronously!");
} else {
CompletableFuture<Void> future = topics.createNonPartitionedTopicAsync(topicUri);
future.whenComplete((unused, throwable)
-> logger.trace("Successfully created non-partitioned topic \"" + topicUri + "\" asynchronously!"))
.exceptionally(ex -> {
logger.error("Failed to create non-partitioned topic \"" + topicUri + "\" asynchronously!");
return null;
});
}
} else {
if (!asyncApi) {
topics.createPartitionedTopic(topicUri, partitionNum);
logger.trace("Successfully created partitioned topic \"" + topicUri + "\"" +
"(partition_num: " + partitionNum + ") synchronously!");
} else {
CompletableFuture<Void> future = topics.createPartitionedTopicAsync(topicUri, partitionNum);
future.whenComplete((unused, throwable)
-> logger.trace("Successfully created partitioned topic \"" + topicUri + "\"" +
"(partition_num: " + partitionNum + ") asynchronously!"))
.exceptionally(ex -> {
logger.error("Failed to create partitioned topic \"" + topicUri + "\"" +
"(partition_num: " + partitionNum + ") asynchronously!");
return null;
});
}
}
}
// Delete the topic
else {
if (!partitionTopic) {
if (!asyncApi) {
topics.delete(topicUri, true);
logger.trace("Successfully deleted non-partitioned topic \"" + topicUri + "\" synchronously!");
} else {
CompletableFuture<Void> future = topics.deleteAsync(topicUri, true);
future.whenComplete((unused, throwable)
-> logger.trace("Successfully deleted non-partitioned topic \"" + topicUri + "\" asynchronously!"))
.exceptionally(ex -> {
logger.error("Failed to delete non-partitioned topic \"" + topicUri + "\" asynchronously!");
return null;
});
}
} else {
if (!asyncApi) {
topics.deletePartitionedTopic(topicUri, true);
logger.trace("Successfully deleted partitioned topic \"" + topicUri + "\" synchronously!");
} else {
CompletableFuture<Void> future = topics.deletePartitionedTopicAsync(topicUri, true);
future.whenComplete((unused, throwable)
-> logger.trace("Successfully deleted partitioned topic \"" + topicUri + "\" asynchronously!"))
.exceptionally(ex -> {
logger.error("Failed to delete partitioned topic \"" + topicUri + "\" asynchronously!");
return null;
});
}
}
}
}
catch (PulsarAdminException e) {
e.printStackTrace();
String errMsg = String.format("Unexpected error when %s pulsar topic: %s (partition topic: %b; partition number: %d)",
(!adminDelOp ? "creating" : "deleting"),
topicUri,
partitionTopic,
partitionNum);
throw new RuntimeException(errMsg);
}
}
}

39
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndMapper.java
View File

@ -1,39 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import java.util.function.LongFunction;
public class PulsarBatchProducerEndMapper extends PulsarOpMapper {
public PulsarBatchProducerEndMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc)
{
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
}
@Override
public PulsarOp apply(long value) {
return new PulsarBatchProducerEndOp();
}
}

50
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndOp.java
View File

@ -1,50 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.api.errors.BasicError;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.common.util.FutureUtil;
import java.util.List;
import java.util.concurrent.CompletableFuture;
public class PulsarBatchProducerEndOp extends SyncPulsarOp {
@Override
public void run() {
List<CompletableFuture<MessageId>> container = PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.get();
Producer<?> producer = PulsarBatchProducerStartOp.threadLocalProducer.get();
if ((container != null) && (!container.isEmpty())) {
try {
// producer.flushAsync().get();
FutureUtil.waitForAll(container).get();
} catch (Exception e) {
throw new RuntimeException("Batch Producer:: failed to send (some of) the batched messages!");
}
container.clear();
PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.set(null);
}
else {
throw new BasicError("You tried to end an empty batch message container. This means you" +
" did initiate the batch container properly, or there is an error in your" +
" pulsar op sequencing and ratios.");
}
}
}

77
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerMapper.java
View File

@ -1,77 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.Map;
import java.util.function.LongFunction;
public class PulsarBatchProducerMapper extends PulsarOpMapper {
private final static Logger logger = LogManager.getLogger(PulsarBatchProducerMapper.class);
private final LongFunction<String> keyFunc;
private final LongFunction<String> propFunc;
private final LongFunction<String> payloadFunc;
public PulsarBatchProducerMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<String> keyFunc,
LongFunction<String> propFunc,
LongFunction<String> payloadFunc) {
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
this.keyFunc = keyFunc;
this.propFunc = propFunc;
this.payloadFunc = payloadFunc;
}
@Override
public PulsarOp apply(long value) {
String msgKey = keyFunc.apply(value);
String msgPayload = payloadFunc.apply(value);
// Check if msgPropJonStr is valid JSON string with a collection of key/value pairs
// - if Yes, convert it to a map
// - otherwise, log an error message and ignore message properties without throwing a runtime exception
Map<String, String> msgProperties = new HashMap<>();
String msgPropJsonStr = propFunc.apply(value);
try {
msgProperties = PulsarActivityUtil.convertJsonToMap(msgPropJsonStr);
}
catch (Exception e) {
logger.error(
"PulsarProducerMapper:: Error parsing message property JSON string {}, ignore message properties!",
msgPropJsonStr);
}
return new PulsarBatchProducerOp(
clientSpace.getPulsarSchema(),
msgKey,
msgProperties,
msgPayload
);
}
}

83
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerOp.java
View File

@ -1,83 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.util.AvroUtil;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.TypedMessageBuilder;
import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
import org.apache.pulsar.common.schema.SchemaType;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
public class PulsarBatchProducerOp extends SyncPulsarOp {
private final Schema<?> pulsarSchema;
private final String msgKey;
private final Map<String, String> msgProperties;
private final String msgPayload;
public PulsarBatchProducerOp(Schema<?> schema,
String key,
Map<String, String> msgProperties,
String payload) {
this.pulsarSchema = schema;
this.msgKey = key;
this.msgProperties = msgProperties;
this.msgPayload = payload;
}
@Override
public void run() {
if ((msgPayload == null) || msgPayload.isEmpty()) {
throw new RuntimeException("Message payload (\"msg-value\") can't be empty!");
}
List<CompletableFuture<MessageId>> container = PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.get();
Producer<?> producer = PulsarBatchProducerStartOp.threadLocalProducer.get();
assert (producer != null) && (container != null);
TypedMessageBuilder typedMessageBuilder = producer.newMessage(pulsarSchema);
if ((msgKey != null) && (!msgKey.isEmpty())) {
typedMessageBuilder = typedMessageBuilder.key(msgKey);
}
if (!msgProperties.isEmpty()) {
typedMessageBuilder = typedMessageBuilder.properties(msgProperties);
}
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro(
(GenericAvroSchema) pulsarSchema,
pulsarSchema.getSchemaInfo().getSchemaDefinition(),
msgPayload
);
typedMessageBuilder = typedMessageBuilder.value(payload);
} else {
typedMessageBuilder = typedMessageBuilder.value(msgPayload.getBytes(StandardCharsets.UTF_8));
}
container.add(typedMessageBuilder.sendAsync());
}
}

44
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartMapper.java
View File

@ -1,44 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.pulsar.client.api.Producer;
import java.util.function.LongFunction;
public class PulsarBatchProducerStartMapper extends PulsarOpMapper {
private final LongFunction<Producer<?>> batchProducerFunc;
public PulsarBatchProducerStartMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Producer<?>> batchProducerFunc) {
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
this.batchProducerFunc = batchProducerFunc;
}
@Override
public PulsarOp apply(long value) {
Producer<?> batchProducer = batchProducerFunc.apply(value);
return new PulsarBatchProducerStartOp(batchProducer);
}
}

49
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartOp.java
View File

@ -1,49 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.api.errors.BasicError;
import org.apache.commons.compress.utils.Lists;
import org.apache.pulsar.client.api.*;
import java.util.List;
import java.util.concurrent.CompletableFuture;
public class PulsarBatchProducerStartOp extends SyncPulsarOp {
// TODO: ensure sane container lifecycle management
public final transient static ThreadLocal<List<CompletableFuture<MessageId>>> threadLocalBatchMsgContainer = new ThreadLocal<>();
public final transient static ThreadLocal<Producer<?>> threadLocalProducer = new ThreadLocal<>();
public PulsarBatchProducerStartOp(Producer<?> batchProducer) {
threadLocalProducer.set(batchProducer);
}
@Override
public void run() {
List<CompletableFuture<MessageId>> container = threadLocalBatchMsgContainer.get();
if (container == null) {
container = Lists.newArrayList();
threadLocalBatchMsgContainer.set(container);
} else {
throw new BasicError("You tried to create a batch message container where one was already" +
" defined. This means you did not flush and unset the last container, or there is an error in your" +
" pulsar op sequencing and ratios.");
}
}
}

104
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java
View File

@ -1,104 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.transaction.Transaction;
import java.util.HashMap;
import java.util.Map;
import java.util.function.LongFunction;
import java.util.function.Supplier;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class PulsarConsumerMapper extends PulsarTransactOpMapper {
private final static Logger logger = LogManager.getLogger(PulsarProducerMapper.class);
private final LongFunction<Consumer<?>> consumerFunc;
private final EndToEndStartingTimeSource endToEndStartingTimeSource;
private final LongFunction<String> payloadRttFieldFunc;
public PulsarConsumerMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> useTransactionFunc,
LongFunction<Boolean> seqTrackingFunc,
LongFunction<Supplier<Transaction>> transactionSupplierFunc,
LongFunction<Consumer<?>> consumerFunc,
EndToEndStartingTimeSource endToEndStartingTimeSource,
LongFunction<String> payloadRttFieldFunc) {
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, useTransactionFunc, seqTrackingFunc, transactionSupplierFunc);
this.consumerFunc = consumerFunc;
this.endToEndStartingTimeSource = endToEndStartingTimeSource;
this.payloadRttFieldFunc = payloadRttFieldFunc;
}
@Override
public PulsarOp apply(long value) {
boolean seqTracking = seqTrackingFunc.apply(value);
Consumer<?> consumer = consumerFunc.apply(value);
boolean asyncApi = asyncApiFunc.apply(value);
boolean useTransaction = useTransactionFunc.apply(value);
Supplier<Transaction> transactionSupplier = transactionSupplierFunc.apply(value);
String payloadRttFieldFunc = this.payloadRttFieldFunc.apply(value);
return new PulsarConsumerOp(
pulsarActivity,
asyncApi,
useTransaction,
seqTracking,
transactionSupplier,
consumer,
clientSpace.getPulsarSchema(),
clientSpace.getPulsarClientConf().getConsumerTimeoutSeconds(),
endToEndStartingTimeSource,
this::getReceivedMessageSequenceTracker,
payloadRttFieldFunc);
}
private ReceivedMessageSequenceTracker getReceivedMessageSequenceTracker(String topicName) {
return receivedMessageSequenceTrackersForTopicThreadLocal.get()
.computeIfAbsent(topicName, k -> createReceivedMessageSequenceTracker());
}
private ReceivedMessageSequenceTracker createReceivedMessageSequenceTracker() {
return new ReceivedMessageSequenceTracker(pulsarActivity.getMsgErrOutOfSeqCounter(),
pulsarActivity.getMsgErrDuplicateCounter(),
pulsarActivity.getMsgErrLossCounter());
}
private final ThreadLocal<Map<String, ReceivedMessageSequenceTracker>> receivedMessageSequenceTrackersForTopicThreadLocal =
ThreadLocal.withInitial(HashMap::new);
}

303
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java
View File

@ -1,303 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Function;
import java.util.function.Supplier;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnexpectedException;
import io.nosqlbench.driver.pulsar.util.AvroUtil;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.*;
import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.api.transaction.Transaction;
import org.apache.pulsar.common.schema.SchemaType;
public class PulsarConsumerOp implements PulsarOp {
private final static Logger logger = LogManager.getLogger(PulsarConsumerOp.class);
private final PulsarActivity pulsarActivity;
private final boolean asyncPulsarOp;
private final boolean useTransaction;
private final boolean seqTracking;
private final Supplier<Transaction> transactionSupplier;
private final Consumer<?> consumer;
private final Schema<?> pulsarSchema;
private final int timeoutSeconds;
private final EndToEndStartingTimeSource endToEndStartingTimeSource;
private final Counter bytesCounter;
private final Histogram messageSizeHistogram;
private final Timer transactionCommitTimer;
// keep track of end-to-end message latency
private final Histogram e2eMsgProcLatencyHistogram;
private final Function<String, ReceivedMessageSequenceTracker> receivedMessageSequenceTrackerForTopic;
private final Histogram payloadRttHistogram;
private final String payloadRttTrackingField;
private org.apache.avro.Schema avroSchema;
public PulsarConsumerOp(
PulsarActivity pulsarActivity,
boolean asyncPulsarOp,
boolean useTransaction,
boolean seqTracking,
Supplier<Transaction> transactionSupplier,
Consumer<?> consumer,
Schema<?> schema,
int timeoutSeconds,
EndToEndStartingTimeSource endToEndStartingTimeSource,
Function<String, ReceivedMessageSequenceTracker> receivedMessageSequenceTrackerForTopic,
String payloadRttTrackingField)
{
this.pulsarActivity = pulsarActivity;
this.asyncPulsarOp = asyncPulsarOp;
this.useTransaction = useTransaction;
this.seqTracking = seqTracking;
this.transactionSupplier = transactionSupplier;
this.consumer = consumer;
this.pulsarSchema = schema;
this.timeoutSeconds = timeoutSeconds;
this.endToEndStartingTimeSource = endToEndStartingTimeSource;
this.bytesCounter = pulsarActivity.getBytesCounter();
this.messageSizeHistogram = pulsarActivity.getMessageSizeHistogram();
this.transactionCommitTimer = pulsarActivity.getCommitTransactionTimer();
this.e2eMsgProcLatencyHistogram = pulsarActivity.getE2eMsgProcLatencyHistogram();
this.payloadRttHistogram = pulsarActivity.getPayloadRttHistogram();
this.receivedMessageSequenceTrackerForTopic = receivedMessageSequenceTrackerForTopic;
this.payloadRttTrackingField = payloadRttTrackingField;
}
private void checkAndUpdateMessageErrorCounter(Message<?> message) {
String msgSeqIdStr = message.getProperty(PulsarActivityUtil.MSG_SEQUENCE_NUMBER);
if ( !StringUtils.isBlank(msgSeqIdStr) ) {
long sequenceNumber = Long.parseLong(msgSeqIdStr);
ReceivedMessageSequenceTracker receivedMessageSequenceTracker = receivedMessageSequenceTrackerForTopic.apply(message.getTopicName());
receivedMessageSequenceTracker.sequenceNumberReceived(sequenceNumber);
}
}
@Override
public void run(Runnable timeTracker) {
final Transaction transaction;
if (useTransaction) {
// if you are in a transaction you cannot set the schema per-message
transaction = transactionSupplier.get();
}
else {
transaction = null;
}
if (!asyncPulsarOp) {
try {
Message<?> message;
if (timeoutSeconds <= 0) {
// wait forever
message = consumer.receive();
}
else {
message = consumer
.receive(timeoutSeconds, TimeUnit.SECONDS);
if (message == null) {
throw new TimeoutException("Did not receive a message within "+timeoutSeconds+" seconds");
}
}
handleMessage(transaction, message);
}
catch (Exception e) {
logger.error(
"Sync message receiving failed - timeout value: {} seconds ", timeoutSeconds, e);
throw new PulsarDriverUnexpectedException("" +
"Sync message receiving failed - timeout value: " + timeoutSeconds + " seconds ");
}
}
else {
try {
CompletableFuture<? extends Message<?>> msgRecvFuture = consumer.receiveAsync();
if (useTransaction) {
// add commit step
msgRecvFuture = msgRecvFuture.thenCompose(msg -> {
Timer.Context ctx = transactionCommitTimer.time();
return transaction
.commit()
.whenComplete((m,e) -> ctx.close())
.thenApply(v-> msg);
}
);
}
msgRecvFuture.thenAccept(message -> {
try {
handleMessage(transaction, message);
} catch (PulsarClientException | TimeoutException e) {
pulsarActivity.asyncOperationFailed(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
pulsarActivity.asyncOperationFailed(e.getCause());
}
}).exceptionally(ex -> {
pulsarActivity.asyncOperationFailed(ex);
return null;
});
}
catch (Exception e) {
throw new PulsarDriverUnexpectedException(e);
}
}
}
private void handleMessage(Transaction transaction, Message<?> message)
throws PulsarClientException, InterruptedException, ExecutionException, TimeoutException {
// acknowledge the message as soon as possible
if (!useTransaction) {
consumer.acknowledgeAsync(message.getMessageId())
.get(timeoutSeconds, TimeUnit.SECONDS);
} else {
consumer.acknowledgeAsync(message.getMessageId(), transaction)
.get(timeoutSeconds, TimeUnit.SECONDS);
// little problem: here we are counting the "commit" time
// inside the overall time spent for the execution of the consume operation
// we should refactor this operation as for PulsarProducerOp, and use the passed callback
// to track with precision the time spent for the operation and for the commit
try (Timer.Context ctx = transactionCommitTimer.time()) {
transaction.commit().get();
}
}
if (logger.isDebugEnabled()) {
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
org.apache.avro.Schema avroSchema = getSchemaFromConfiguration();
org.apache.avro.generic.GenericRecord avroGenericRecord =
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData());
logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}",
consumer.getConsumerName(),
message.getKey(),
message.getProperties(),
avroGenericRecord.toString());
}
else {
logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}",
consumer.getConsumerName(),
message.getKey(),
message.getProperties(),
new String(message.getData()));
}
}
if (!payloadRttTrackingField.isEmpty()) {
Object decodedPayload = message.getValue();
Long extractedSendTime = null;
// if Pulsar is able to decode this it is better to let it do the work
// because Pulsar caches the Schema, handles Schema evolution
// as much efficiently as possible
if (decodedPayload instanceof GenericRecord) {
GenericRecord pulsarGenericRecord = (GenericRecord) decodedPayload;
Object field = pulsarGenericRecord.getField(payloadRttTrackingField);
if (field != null) {
if (field instanceof Number) {
extractedSendTime = ((Number) field).longValue();
} else {
extractedSendTime = Long.valueOf(field.toString());
}
}
} else {
org.apache.avro.Schema avroSchema = getSchemaFromConfiguration();
org.apache.avro.generic.GenericRecord avroGenericRecord =
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData());
if (avroGenericRecord.hasField(payloadRttTrackingField)) {
extractedSendTime = (Long) avroGenericRecord.get(payloadRttTrackingField);
}
}
if (extractedSendTime != null) {
long delta = System.currentTimeMillis() - extractedSendTime;
payloadRttHistogram.update(delta);
}
}
// keep track end-to-end message processing latency
if (endToEndStartingTimeSource != EndToEndStartingTimeSource.NONE) {
long startTimeStamp = 0L;
switch (endToEndStartingTimeSource) {
case MESSAGE_PUBLISH_TIME:
startTimeStamp = message.getPublishTime();
break;
case MESSAGE_EVENT_TIME:
startTimeStamp = message.getEventTime();
break;
case MESSAGE_PROPERTY_E2E_STARTING_TIME:
String startingTimeProperty = message.getProperty("e2e_starting_time");
startTimeStamp = startingTimeProperty != null ? Long.parseLong(startingTimeProperty) : 0L;
break;
}
if (startTimeStamp != 0L) {
long e2eMsgLatency = System.currentTimeMillis() - startTimeStamp;
e2eMsgProcLatencyHistogram.update(e2eMsgLatency);
}
}
// keep track of message errors and update error counters
if (seqTracking) checkAndUpdateMessageErrorCounter(message);
int messageSize = message.getData().length;
bytesCounter.inc(messageSize);
messageSizeHistogram.update(messageSize);
}
private org.apache.avro.Schema getSchemaFromConfiguration() {
String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition();
// no need for synchronization, this is only a cache
// in case of the race we will parse the string twice, not a big
if (avroSchema == null) {
avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
}
return avroSchema;
}
}

30
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOp.java
View File

@ -1,30 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
/**
* Base type of all Pulsar Operations including Producers and Consumers.
*/
public interface PulsarOp {
/**
* Execute the operation, invoke the timeTracker when the operation ended.
* The timeTracker can be invoked in a separate thread, it is only used for metrics.
* @param timeTracker
*/
void run(Runnable timeTracker);
}

45
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOpMapper.java
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.transaction.Transaction;
import java.util.function.LongFunction;
import java.util.function.Supplier;
public abstract class PulsarOpMapper implements LongFunction<PulsarOp> {
protected final CommandTemplate cmdTpl;
protected final PulsarSpace clientSpace;
protected final PulsarActivity pulsarActivity;
protected final LongFunction<Boolean> asyncApiFunc;
public PulsarOpMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc)
{
this.cmdTpl = cmdTpl;
this.clientSpace = clientSpace;
this.pulsarActivity = pulsarActivity;
this.asyncApiFunc = asyncApiFunc;
}
}

129
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerMapper.java
View File

@ -1,129 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.function.LongFunction;
import java.util.function.Supplier;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.transaction.Transaction;
/**
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
* retried if needed.
*
* This function doesn't act *as* the operation. It merely maps the construction logic into
* a simple functional type, given the component functions.
*
* For additional parameterization, the command template is also provided.
*/
public class PulsarProducerMapper extends PulsarTransactOpMapper {
private final static Logger logger = LogManager.getLogger(PulsarProducerMapper.class);
private final LongFunction<Producer<?>> producerFunc;
private final Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> seqErrSimuTypes;
private final LongFunction<String> keyFunc;
private final LongFunction<String> propFunc;
private final LongFunction<String> payloadFunc;
public PulsarProducerMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> useTransactionFunc,
LongFunction<Boolean> seqTrackingFunc,
LongFunction<Supplier<Transaction>> transactionSupplierFunc,
LongFunction<Producer<?>> producerFunc,
Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> seqErrSimuTypes,
LongFunction<String> keyFunc,
LongFunction<String> propFunc,
LongFunction<String> payloadFunc) {
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, useTransactionFunc, seqTrackingFunc, transactionSupplierFunc);
this.producerFunc = producerFunc;
this.seqErrSimuTypes = seqErrSimuTypes;
this.keyFunc = keyFunc;
this.propFunc = propFunc;
this.payloadFunc = payloadFunc;
}
@Override
public PulsarOp apply(long value) {
boolean asyncApi = asyncApiFunc.apply(value);
boolean useTransaction = useTransactionFunc.apply(value);
Supplier<Transaction> transactionSupplier = transactionSupplierFunc.apply(value);
Producer<?> producer = producerFunc.apply(value);
String msgKey = keyFunc.apply(value);
String msgPayload = payloadFunc.apply(value);
// Check if msgPropJonStr is valid JSON string with a collection of key/value pairs
// - if Yes, convert it to a map
// - otherwise, log an error message and ignore message properties without throwing a runtime exception
Map<String, String> msgProperties = new HashMap<>();
String msgPropJsonStr = propFunc.apply(value);
if (!StringUtils.isBlank(msgPropJsonStr)) {
try {
msgProperties = PulsarActivityUtil.convertJsonToMap(msgPropJsonStr);
} catch (Exception e) {
logger.error(
"Error parsing message property JSON string {}, ignore message properties!",
msgPropJsonStr);
}
}
boolean sequenceTrackingEnabled = seqTrackingFunc.apply(value);
if (sequenceTrackingEnabled) {
long nextSequenceNumber = getMessageSequenceNumberSendingHandler(producer.getTopic())
.getNextSequenceNumber(seqErrSimuTypes);
msgProperties.put(PulsarActivityUtil.MSG_SEQUENCE_NUMBER, String.valueOf(nextSequenceNumber));
}
return new PulsarProducerOp(
pulsarActivity,
asyncApi,
useTransaction,
transactionSupplier,
producer,
clientSpace.getPulsarSchema(),
msgKey,
msgProperties,
msgPayload);
}
private MessageSequenceNumberSendingHandler getMessageSequenceNumberSendingHandler(String topicName) {
return MessageSequenceNumberSendingHandlersThreadLocal.get()
.computeIfAbsent(topicName, k -> new MessageSequenceNumberSendingHandler());
}
private final ThreadLocal<Map<String, MessageSequenceNumberSendingHandler>> MessageSequenceNumberSendingHandlersThreadLocal =
ThreadLocal.withInitial(HashMap::new);
}

304
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerOp.java
View File

@ -1,304 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException;
import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnexpectedException;
import io.nosqlbench.driver.pulsar.util.AvroUtil;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.*;
import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.api.schema.KeyValueSchema;
import org.apache.pulsar.client.api.transaction.Transaction;
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
import org.apache.pulsar.common.schema.KeyValue;
import org.apache.pulsar.common.schema.SchemaType;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.function.Supplier;
public class PulsarProducerOp implements PulsarOp {
private final static Logger logger = LogManager.getLogger(PulsarProducerOp.class);
private final PulsarActivity pulsarActivity;
private final boolean asyncPulsarOp;
private final boolean useTransaction;
private final Supplier<Transaction> transactionSupplier;
private final Producer<?> producer;
private final Schema<?> pulsarSchema;
private final String msgKey;
private final Map<String, String> msgProperties;
private final String msgPayload;
private final Counter bytesCounter;
private final Histogram messageSizeHistogram;
private final Timer transactionCommitTimer;
private org.apache.avro.Schema avroSchema;
private org.apache.avro.Schema avroKeySchema;
public PulsarProducerOp( PulsarActivity pulsarActivity,
boolean asyncPulsarOp,
boolean useTransaction,
Supplier<Transaction> transactionSupplier,
Producer<?> producer,
Schema<?> schema,
String key,
Map<String, String> msgProperties,
String payload) {
this.pulsarActivity = pulsarActivity;
this.asyncPulsarOp = asyncPulsarOp;
this.useTransaction = useTransaction;
this.transactionSupplier = transactionSupplier;
this.producer = producer;
this.pulsarSchema = schema;
this.msgKey = key;
this.msgProperties = msgProperties;
this.msgPayload = payload;
this.bytesCounter = pulsarActivity.getBytesCounter();
this.messageSizeHistogram = pulsarActivity.getMessageSizeHistogram();
this.transactionCommitTimer = pulsarActivity.getCommitTransactionTimer();
}
@Override
public void run(Runnable timeTracker) {
if ( StringUtils.isBlank(msgPayload)) {
throw new PulsarDriverParamException("Message payload (\"msg-value\") can't be empty!");
}
TypedMessageBuilder typedMessageBuilder;
final Transaction transaction;
if (useTransaction) {
// if you are in a transaction you cannot set the schema per-message
transaction = transactionSupplier.get();
typedMessageBuilder = producer.newMessage(transaction);
}
else {
transaction = null;
typedMessageBuilder = producer.newMessage(pulsarSchema);
}
// set message key
if (!StringUtils.isBlank(msgKey)) {
typedMessageBuilder = typedMessageBuilder.key(msgKey);
}
// set message properties
if ( !msgProperties.isEmpty() ) {
typedMessageBuilder = typedMessageBuilder.properties(msgProperties);
}
// set message payload
int messageSize;
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
if (pulsarSchema instanceof KeyValueSchema) {
// {KEY IN JSON}||{VALUE IN JSON}
int separator = msgPayload.indexOf("}||{");
if (separator < 0) {
throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)");
}
String keyInput = msgPayload.substring(0, separator + 1);
String valueInput = msgPayload.substring(separator + 3);
KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema;
org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro(
(GenericAvroSchema) keyValueSchema.getValueSchema(),
avroSchema,
valueInput
);
org.apache.avro.Schema avroSchemaForKey = getKeyAvroSchemaFromConfiguration();
GenericRecord key = AvroUtil.GetGenericRecord_PulsarAvro(
(GenericAvroSchema) keyValueSchema.getKeySchema(),
avroSchemaForKey,
keyInput
);
typedMessageBuilder = typedMessageBuilder.value(new KeyValue(key, payload));
// TODO: add a way to calculate the message size for KEY_VALUE messages
messageSize = msgPayload.length();
} else if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro(
(GenericAvroSchema) pulsarSchema,
pulsarSchema.getSchemaInfo().getSchemaDefinition(),
msgPayload
);
typedMessageBuilder = typedMessageBuilder.value(payload);
// TODO: add a way to calculate the message size for AVRO messages
messageSize = msgPayload.length();
} else {
byte[] array = msgPayload.getBytes(StandardCharsets.UTF_8);
typedMessageBuilder = typedMessageBuilder.value(array);
messageSize = array.length;
}
messageSizeHistogram.update(messageSize);
bytesCounter.inc(messageSize);
//TODO: add error handling with failed message production
if (!asyncPulsarOp) {
try {
logger.trace("Sending message");
typedMessageBuilder.send();
if (useTransaction) {
try (Timer.Context ctx = transactionCommitTimer.time()) {
transaction.commit().get();
}
}
if (logger.isDebugEnabled()) {
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
org.apache.avro.generic.GenericRecord avroGenericRecord =
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload);
logger.debug("({}) Sync message sent: msg-key={}; msg-properties={}; msg-payload={})",
producer.getProducerName(),
msgKey,
msgProperties,
avroGenericRecord.toString());
}
else {
logger.debug("({}) Sync message sent; msg-key={}; msg-properties={}; msg-payload={}",
producer.getProducerName(),
msgKey,
msgProperties,
msgPayload);
}
}
}
catch (PulsarClientException | ExecutionException | InterruptedException pce) {
String errMsg =
"Sync message sending failed: " +
"key - " + msgKey + "; " +
"properties - " + msgProperties + "; " +
"payload - " + msgPayload;
logger.trace(errMsg);
throw new PulsarDriverUnexpectedException(errMsg);
}
timeTracker.run();
}
else {
try {
// we rely on blockIfQueueIsFull in order to throttle the request in this case
CompletableFuture<?> future = typedMessageBuilder.sendAsync();
if (useTransaction) {
// add commit step
future = future.thenCompose(msg -> {
Timer.Context ctx = transactionCommitTimer.time();
return transaction
.commit()
.whenComplete((m,e) -> ctx.close())
.thenApply(v-> msg);
}
);
}
future.whenComplete((messageId, error) -> {
if (logger.isDebugEnabled()) {
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
org.apache.avro.generic.GenericRecord avroGenericRecord =
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload);
logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={})",
producer.getProducerName(),
msgKey,
msgProperties,
avroGenericRecord.toString());
}
else {
logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={}",
producer.getProducerName(),
msgKey,
msgProperties,
msgPayload);
}
}
timeTracker.run();
}).exceptionally(ex -> {
logger.error("Async message sending failed: " +
"key - " + msgKey + "; " +
"properties - " + msgProperties + "; " +
"payload - " + msgPayload);
pulsarActivity.asyncOperationFailed(ex);
return null;
});
}
catch (Exception e) {
throw new PulsarDriverUnexpectedException(e);
}
}
}
private org.apache.avro.Schema getAvroSchemaFromConfiguration() {
// no need for synchronization, this is only a cache
// in case of the race we will parse the string twice, not a big
if (avroSchema == null) {
if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
Schema valueSchema = kvSchema.getValueSchema();
String avroDefStr = valueSchema.getSchemaInfo().getSchemaDefinition();
avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
} else {
String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition();
avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
}
}
return avroSchema;
}
private org.apache.avro.Schema getKeyAvroSchemaFromConfiguration() {
// no need for synchronization, this is only a cache
// in case of the race we will parse the string twice, not a big
if (avroKeySchema == null) {
if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
Schema keySchema = kvSchema.getKeySchema();
String avroDefStr = keySchema.getSchemaInfo().getSchemaDefinition();
avroKeySchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
} else {
throw new RuntimeException("We are not using KEY_VALUE schema, so no Schema for the Key!");
}
}
return avroKeySchema;
}
}

51
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderMapper.java
View File

@ -1,51 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.pulsar.client.api.Reader;
import java.util.function.LongFunction;
public class PulsarReaderMapper extends PulsarOpMapper {
private final LongFunction<Reader<?>> readerFunc;
public PulsarReaderMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Reader<?>> readerFunc)
{
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
this.readerFunc = readerFunc;
}
@Override
public PulsarOp apply(long value) {
Reader<?> reader = readerFunc.apply(value);
boolean asyncApi = asyncApiFunc.apply(value);
return new PulsarReaderOp(
reader,
clientSpace.getPulsarSchema(),
asyncApi
);
}
}

76
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderOp.java
View File

@ -1,76 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.util.AvroUtil;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Reader;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.common.schema.SchemaType;
public class PulsarReaderOp extends SyncPulsarOp {
private final Reader<?> reader;
private final Schema<?> pulsarSchema;
private final boolean asyncPulsarOp;
public PulsarReaderOp(Reader<?> reader, Schema<?> schema, boolean asyncPulsarOp) {
this.reader = reader;
this.pulsarSchema = schema;
this.asyncPulsarOp = asyncPulsarOp;
}
public void syncRead() {
try {
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition();
// TODO: how many messages to read per NB cycle?
Message<?> message;
while (reader.hasMessageAvailable()) {
message = reader.readNext();
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
org.apache.avro.Schema avroSchema =
AvroUtil.GetSchema_ApacheAvro(avroDefStr);
org.apache.avro.generic.GenericRecord avroGenericRecord =
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData());
System.out.println("msg-key=" + message.getKey() + " msg-payload=" + avroGenericRecord.toString());
} else {
System.out.println("msg-key=" + message.getKey() + " msg-payload=" + new String(message.getData()));
}
}
} catch (PulsarClientException e) {
throw new RuntimeException(e);
}
}
public void asyncRead() {
//TODO: add support for async read
}
@Override
public void run() {
if (!asyncPulsarOp)
syncRead();
else
asyncRead();
}
}

45
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarTransactOpMapper.java
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.pulsar.client.api.transaction.Transaction;
import java.util.function.LongFunction;
import java.util.function.Supplier;
public abstract class PulsarTransactOpMapper extends PulsarOpMapper {
protected final LongFunction<Boolean> useTransactionFunc;
protected final LongFunction<Boolean> seqTrackingFunc;
protected final LongFunction<Supplier<Transaction>> transactionSupplierFunc;
public PulsarTransactOpMapper(CommandTemplate cmdTpl,
PulsarSpace clientSpace,
PulsarActivity pulsarActivity,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> useTransactionFunc,
LongFunction<Boolean> seqTrackingFunc,
LongFunction<Supplier<Transaction>> transactionSupplierFunc)
{
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
this.useTransactionFunc = useTransactionFunc;
this.seqTrackingFunc = seqTrackingFunc;
this.transactionSupplierFunc = transactionSupplierFunc;
}
}

541
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReadyPulsarOp.java
View File

@ -1,541 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import io.nosqlbench.driver.pulsar.PulsarActivity;
import io.nosqlbench.driver.pulsar.PulsarSpace;
import io.nosqlbench.driver.pulsar.PulsarSpaceCache;
import io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException;
import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnsupportedOpException;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.engine.api.templating.CommandTemplate;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.Reader;
import org.apache.pulsar.client.api.transaction.Transaction;
import java.util.*;
import java.util.function.LongFunction;
import java.util.function.Supplier;
import java.util.stream.Collectors;
public class ReadyPulsarOp extends BaseOpDispenser<PulsarOp> {
// TODO: Add this to the pulsar driver docs
public static final String RTT_TRACKING_FIELD = "payload-tracking-field";
private final static Logger logger = LogManager.getLogger(ReadyPulsarOp.class);
private final OpTemplate opTpl;
private final CommandTemplate cmdTpl;
private final PulsarSpace clientSpace;
private final LongFunction<PulsarOp> opFunc;
private final PulsarActivity pulsarActivity;
// TODO: Add docs for the command template with respect to the OpTemplate
public ReadyPulsarOp(OpTemplate optpl, PulsarSpaceCache pcache, PulsarActivity pulsarActivity) {
super(optpl);
// TODO: Consider parsing map structures into equivalent binding representation
this.pulsarActivity = pulsarActivity;
this.opTpl = optpl;
this.cmdTpl = new CommandTemplate(optpl);
// TODO: At the moment, only supports static "client"
String client_name = lookupStaticParameter("client", false, "default");
this.clientSpace = pcache.getPulsarSpace(client_name);
this.opFunc = resolve();
}
@Override
public PulsarOp apply(long value) {
return opFunc.apply(value);
}
private LongFunction<PulsarOp> resolve() {
String stmtOpType = lookupStaticParameter("optype", true, null);
if (cmdTpl.containsKey("topic_url")) {
throw new PulsarDriverParamException("[resolve()] \"topic_url\" parameter is not valid. Perhaps you mean \"topic_uri\"?");
}
// Doc-level parameter: topic_uri
LongFunction<String> topicUriFunc = lookupParameterFunc(PulsarActivityUtil.DOC_LEVEL_PARAMS.TOPIC_URI.label);
logger.info("topic_uri: {}", topicUriFunc.apply(0));
// Doc-level parameter: async_api
boolean useAsyncApi = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ASYNC_API.label, false, "false"));
LongFunction<Boolean> asyncApiFunc = (l) -> useAsyncApi;
logger.info("async_api: {}", useAsyncApi);
// Doc-level parameter: use_transaction
boolean useTransaction = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.USE_TRANSACTION.label, false, "false"));
LongFunction<Boolean> useTransactionFunc = (l) -> useTransaction;
logger.info("use_transaction: {}", useTransaction);
// Doc-level parameter: admin_delop
boolean adminDelOp = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ADMIN_DELOP.label, false, "false"));
LongFunction<Boolean> adminDelOpFunc = (l) -> adminDelOp;
logger.info("admin_delop: {}", adminDelOp);
// Doc-level parameter: seq_tracking
boolean seqTracking = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.SEQ_TRACKING.label, false, "false"));
LongFunction<Boolean> seqTrackingFunc = (l) -> seqTracking;
logger.info("seq_tracking: {}", seqTracking);
// TODO: Collapse this pattern into a simple version and flatten out all call sites
LongFunction<String> payloadRttFieldFunc = lookupParameterFunc(RTT_TRACKING_FIELD, false, "");
logger.info("payload_rtt_field_func: {}", payloadRttFieldFunc.apply(0));
// TODO: Complete implementation for websocket-producer and managed-ledger
// Admin operation: create/delete tenant
if ( StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TENANT.label) ) {
return resolveAdminTenant(clientSpace, asyncApiFunc, adminDelOpFunc);
}
// Admin operation: create/delete namespace
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_NAMESPACE.label)) {
return resolveAdminNamespace(clientSpace, asyncApiFunc, adminDelOpFunc);
}
// Admin operation: create/delete topic
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TOPIC.label)) {
return resolveAdminTopic(clientSpace, topicUriFunc, asyncApiFunc, adminDelOpFunc);
}
// Regular/non-admin operation: single message sending (producer)
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_SEND.label)) {
return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc);
}
// Regular/non-admin operation: single message consuming from a single topic (consumer)
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_CONSUME.label)) {
return resolveMsgConsume(
clientSpace,
topicUriFunc,
asyncApiFunc,
useTransactionFunc,
seqTrackingFunc,
parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE),
payloadRttFieldFunc);
}
// Regular/non-admin operation: single message consuming from multiple-topics (consumer)
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_MULTI_CONSUME.label)) {
return resolveMultiTopicMsgConsume(
clientSpace,
topicUriFunc,
asyncApiFunc,
useTransactionFunc,
seqTrackingFunc,
payloadRttFieldFunc);
}
// Regular/non-admin operation: single message consuming a single topic (reader)
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_READ.label)) {
return resolveMsgRead(clientSpace, topicUriFunc, asyncApiFunc);
}
// // Regular/non-admin operation: batch message processing - batch start
// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_START.label)) {
// return resolveMsgBatchSendStart(clientSpace, topicUriFunc, asyncApiFunc);
// }
// // Regular/non-admin operation: batch message processing - message sending (producer)
// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND.label)) {
// return resolveMsgBatchSend(clientSpace, asyncApiFunc);
// }
// // Regular/non-admin operation: batch message processing - batch send
// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_END.label)) {
// return resolveMsgBatchSendEnd(clientSpace, asyncApiFunc);
// }
// Regular/non-admin operation: end-to-end message processing - sending message
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_SEND.label)) {
return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc);
}
// Regular/non-admin operation: end-to-end message processing - consuming message
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_CONSUME.label)) {
return resolveMsgConsume(
clientSpace,
topicUriFunc,
asyncApiFunc,
useTransactionFunc,
seqTrackingFunc,
parseEndToEndStartingTimeSourceParameter(
EndToEndStartingTimeSource.MESSAGE_PUBLISH_TIME),
payloadRttFieldFunc);
}
// Invalid operation type
else {
throw new PulsarDriverUnsupportedOpException();
}
}
private EndToEndStartingTimeSource parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource defaultValue) {
EndToEndStartingTimeSource endToEndStartingTimeSource = defaultValue;
if (cmdTpl.isStatic(PulsarActivityUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label)) {
endToEndStartingTimeSource = EndToEndStartingTimeSource.valueOf(cmdTpl.getStatic(PulsarActivityUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label).toUpperCase());
}
return endToEndStartingTimeSource;
}
// Admin API: create tenant
private LongFunction<PulsarOp> resolveAdminTenant(
PulsarSpace clientSpace,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc)
{
// "admin_roles" includes comma-separated admin roles:
// e.g. role1, role2
Set<String> roleSet = lookupStaticParameterSet("admin_roles");
LongFunction<Set<String>> adminRolesFunc = (l) -> roleSet;
// "allowed_cluster" includes comma-separated cluster names:
// e.g. cluster1, cluster2
Set<String> clusterSet = lookupStaticParameterSet("allowed_clusters");
LongFunction<Set<String>> allowedClustersFunc = (l) -> clusterSet;
LongFunction<String> tenantFunc = lookupParameterFunc("tenant");
return new PulsarAdminTenantMapper(
cmdTpl,
clientSpace,
pulsarActivity,
asyncApiFunc,
adminDelOpFunc,
adminRolesFunc,
allowedClustersFunc,
tenantFunc);
}
private Set<String> lookupStaticParameterSet(String parameterName) {
return Optional.ofNullable(lookupStaticParameter(parameterName))
.map(value -> {
Set<String> set = Arrays.stream(value.split(","))
.map(String::trim)
.collect(Collectors.toCollection(LinkedHashSet::new));
return set;
}).orElse(Collections.emptySet());
}
// Admin API: create tenant
private LongFunction<PulsarOp> resolveAdminNamespace(
PulsarSpace clientSpace,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc)
{
LongFunction<String> namespaceFunc = lookupParameterFunc("namespace");
return new PulsarAdminNamespaceMapper(
cmdTpl,
clientSpace,
pulsarActivity,
asyncApiFunc,
adminDelOpFunc,
namespaceFunc);
}
// Admin API: create partitioned topic
private LongFunction<PulsarOp> resolveAdminTopic(
PulsarSpace clientSpace,
LongFunction<String> topic_uri_fun,
LongFunction<Boolean> asyncApiFunc,
LongFunction<Boolean> adminDelOpFunc
) {
LongFunction<String> enablePartionFunc = lookupParameterFunc("enable_partition");
LongFunction<String> partitionNumFunc = lookupParameterFunc("partition_num");
return new PulsarAdminTopicMapper(
cmdTpl,
clientSpace,
pulsarActivity,
asyncApiFunc,
adminDelOpFunc,
topic_uri_fun,
enablePartionFunc,
partitionNumFunc);
}
private LongFunction<PulsarOp> resolveMsgSend(
PulsarSpace clientSpace,
LongFunction<String> topic_uri_func,
LongFunction<Boolean> async_api_func,
LongFunction<Boolean> useTransactionFunc,
LongFunction<Boolean> seqTrackingFunc
) {
LongFunction<Supplier<Transaction>> transactionSupplierFunc =
(l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle?
LongFunction<String> cycle_producer_name_func = lookupParameterFunc("producer_name");
LongFunction<Producer<?>> producerFunc =
(l) -> clientSpace.getProducer(topic_uri_func.apply(l), cycle_producer_name_func.apply(l));
// check if we're going to simulate producer message out-of-sequence error
// - message ordering
// - message loss
Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> seqErrSimuTypes = parseSimulatedErrorTypes(lookupStaticParameter("seqerr_simu"));
// message key
LongFunction<String> keyFunc = lookupParameterFunc("msg_key");
// message property
LongFunction<String> propFunc = lookupParameterFunc("msg_property");
LongFunction<String> valueFunc = lookupParameterFunc("msg_value", true);
return new PulsarProducerMapper(
cmdTpl,
clientSpace,
pulsarActivity,
async_api_func,
useTransactionFunc,
seqTrackingFunc,
transactionSupplierFunc,
producerFunc,
seqErrSimuTypes,
keyFunc,
propFunc,
valueFunc);
}
private Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> parseSimulatedErrorTypes(String sequenceErrorSimulatedTypeString) {
if (StringUtils.isBlank(sequenceErrorSimulatedTypeString)) {
return Collections.emptySet();
}
return Arrays.stream(StringUtils.split(sequenceErrorSimulatedTypeString, ','))
.map(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE::parseSimuType)
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toCollection(LinkedHashSet::new));
}
private LongFunction<PulsarOp> resolveMsgConsume(
PulsarSpace clientSpace,
LongFunction<String> topic_uri_func,
LongFunction<Boolean> async_api_func,
LongFunction<Boolean> useTransactionFunc,
LongFunction<Boolean> seqTrackingFunc,
EndToEndStartingTimeSource endToEndStartingTimeSource,
LongFunction<String> rttTrackingFieldFunc
) {
LongFunction<String> subscription_name_func = lookupParameterFunc("subscription_name");
LongFunction<String> subscription_type_func = lookupParameterFunc("subscription_type");
LongFunction<String> consumer_name_func = lookupParameterFunc("consumer_name");
LongFunction<String> ranges_func = lookupParameterFunc("ranges", false, "");
LongFunction<Supplier<Transaction>> transactionSupplierFunc =
(l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle?
LongFunction<Consumer<?>> consumerFunc = (l) ->
clientSpace.getConsumer(
topic_uri_func.apply(l),
subscription_name_func.apply(l),
subscription_type_func.apply(l),
consumer_name_func.apply(l),
ranges_func.apply(l)
);
return new PulsarConsumerMapper(
cmdTpl,
clientSpace,
pulsarActivity,
async_api_func,
useTransactionFunc,
seqTrackingFunc,
transactionSupplierFunc,
consumerFunc,
endToEndStartingTimeSource,
rttTrackingFieldFunc);
}
private LongFunction<PulsarOp> resolveMultiTopicMsgConsume(
PulsarSpace clientSpace,
LongFunction<String> topic_uri_func,
LongFunction<Boolean> async_api_func,
LongFunction<Boolean> useTransactionFunc,
LongFunction<Boolean> seqTrackingFunc,
LongFunction<String> payloadRttFieldFunc
) {
// Topic list (multi-topic)
LongFunction<String> topic_names_func = lookupParameterFunc("topic_names");
// Topic pattern (multi-topic)
LongFunction<String> topics_pattern_func = lookupParameterFunc("topics_pattern");
LongFunction<String> subscription_name_func = lookupParameterFunc("subscription_name");
LongFunction<String> subscription_type_func = lookupParameterFunc("subscription_type");
LongFunction<String> consumer_name_func = lookupParameterFunc("consumer_name");
LongFunction<Supplier<Transaction>> transactionSupplierFunc =
(l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle?
LongFunction<Consumer<?>> mtConsumerFunc = (l) ->
clientSpace.getMultiTopicConsumer(
topic_uri_func.apply(l),
topic_names_func.apply(l),
topics_pattern_func.apply(l),
subscription_name_func.apply(l),
subscription_type_func.apply(l),
consumer_name_func.apply(l)
);
return new PulsarConsumerMapper(
cmdTpl,
clientSpace,
pulsarActivity,
async_api_func,
useTransactionFunc,
seqTrackingFunc,
transactionSupplierFunc,
mtConsumerFunc,
parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE),
payloadRttFieldFunc);
}
private LongFunction<String> lookupParameterFunc(String parameterName) {
return lookupParameterFunc(parameterName, false, null);
}
private LongFunction<String> lookupParameterFunc(String parameterName, boolean required) {
return lookupParameterFunc(parameterName, required, null);
}
private LongFunction<String> lookupParameterFunc(String parameterName, boolean required, String defaultValue) {
if (cmdTpl.containsKey(parameterName)) {
LongFunction<String> lookupFunc;
if (cmdTpl.isStatic(parameterName)) {
String staticValue = cmdTpl.getStatic(parameterName);
lookupFunc = (l) -> staticValue;
} else if (cmdTpl.isDynamic(parameterName)) {
lookupFunc = (l) -> cmdTpl.getDynamic(parameterName, l);
} else {
lookupFunc = (l) -> defaultValue;
}
return lookupFunc;
} else {
if (required) {
throw new PulsarDriverParamException("\"" + parameterName + "\" field must be specified!");
} else {
return (l) -> defaultValue;
}
}
}
private String lookupStaticParameter(String parameterName) {
return lookupStaticParameter(parameterName, false, null);
}
private String lookupStaticParameter(String parameterName, boolean required, String defaultValue) {
if (cmdTpl.containsKey(parameterName)) {
if (cmdTpl.isStatic(parameterName)) {
return cmdTpl.getStatic(parameterName);
} else if (cmdTpl.isDynamic(parameterName)) {
throw new PulsarDriverParamException("\"" + parameterName + "\" parameter must be static");
} else {
return defaultValue;
}
} else {
if (required) {
throw new PulsarDriverParamException("\"" + parameterName + "\" field must be specified!");
} else {
return defaultValue;
}
}
}
private LongFunction<Boolean> toBooleanFunc(LongFunction<String> parameterFunc) {
return (l) -> BooleanUtils.toBoolean(parameterFunc.apply(l));
}
private LongFunction<PulsarOp> resolveMsgRead(
PulsarSpace clientSpace,
LongFunction<String> topic_uri_func,
LongFunction<Boolean> async_api_func
) {
LongFunction<String> reader_name_func = lookupParameterFunc("reader_name");
LongFunction<String> start_msg_pos_str_func = lookupParameterFunc("start_msg_position");
LongFunction<Reader<?>> readerFunc = (l) ->
clientSpace.getReader(
topic_uri_func.apply(l),
reader_name_func.apply(l),
start_msg_pos_str_func.apply(l)
);
return new PulsarReaderMapper(
cmdTpl,
clientSpace,
pulsarActivity,
async_api_func,
readerFunc);
}
private LongFunction<PulsarOp> resolveMsgBatchSendStart(
PulsarSpace clientSpace,
LongFunction<String> topic_uri_func,
LongFunction<Boolean> asyncApiFunc)
{
LongFunction<String> cycle_batch_producer_name_func = lookupParameterFunc("batch_producer_name");
LongFunction<Producer<?>> batchProducerFunc =
(l) -> clientSpace.getProducer(topic_uri_func.apply(l), cycle_batch_producer_name_func.apply(l));
return new PulsarBatchProducerStartMapper(
cmdTpl,
clientSpace,
pulsarActivity,
asyncApiFunc,
batchProducerFunc);
}
private LongFunction<PulsarOp> resolveMsgBatchSend(PulsarSpace clientSpace,
LongFunction<Boolean> asyncApiFunc)
{
LongFunction<String> keyFunc = lookupParameterFunc("msg_key");
// message property
LongFunction<String> propFunc = lookupParameterFunc("msg_property");
LongFunction<String> valueFunc = lookupParameterFunc("msg_value", true);
return new PulsarBatchProducerMapper(
cmdTpl,
clientSpace,
pulsarActivity,
asyncApiFunc,
keyFunc,
propFunc,
valueFunc);
}
private LongFunction<PulsarOp> resolveMsgBatchSendEnd(PulsarSpace clientSpace,
LongFunction<Boolean> asyncApiFunc)
{
return new PulsarBatchProducerEndMapper(
cmdTpl,
clientSpace,
pulsarActivity,
asyncApiFunc);
}
}

166
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTracker.java
View File

@ -1,166 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import com.codahale.metrics.Counter;
import java.util.Iterator;
import java.util.SortedSet;
import java.util.TreeSet;
/**
* Detects message loss, message duplication and out-of-order message delivery
* based on a monotonic sequence number that each received message contains.
* <p>
* Out-of-order messages are detected with a maximum look behind of 1000 sequence number entries.
* This is currently defined as a constant, {@link ReceivedMessageSequenceTracker#DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS}.
*/
class ReceivedMessageSequenceTracker implements AutoCloseable {
private static final int DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS = 1000;
private static final int DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS = 1000;
// message out-of-sequence error counter
private final Counter msgErrOutOfSeqCounter;
// duplicate message error counter
private final Counter msgErrDuplicateCounter;
// message loss error counter
private final Counter msgErrLossCounter;
private final SortedSet<Long> pendingOutOfSeqNumbers;
private final int maxTrackOutOfOrderSequenceNumbers;
private final SortedSet<Long> skippedSeqNumbers;
private final int maxTrackSkippedSequenceNumbers;
private long expectedNumber = -1;
public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter) {
this(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter,
DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS, DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS);
}
public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter,
int maxTrackOutOfOrderSequenceNumbers, int maxTrackSkippedSequenceNumbers) {
this.msgErrOutOfSeqCounter = msgErrOutOfSeqCounter;
this.msgErrDuplicateCounter = msgErrDuplicateCounter;
this.msgErrLossCounter = msgErrLossCounter;
this.maxTrackOutOfOrderSequenceNumbers = maxTrackOutOfOrderSequenceNumbers;
this.maxTrackSkippedSequenceNumbers = maxTrackSkippedSequenceNumbers;
this.pendingOutOfSeqNumbers = new TreeSet<>();
this.skippedSeqNumbers = new TreeSet<>();
}
/**
* Notifies the tracker about a received sequence number
*
* @param sequenceNumber the sequence number of the received message
*/
public void sequenceNumberReceived(long sequenceNumber) {
if (expectedNumber == -1) {
expectedNumber = sequenceNumber + 1;
return;
}
if (sequenceNumber < expectedNumber) {
if (skippedSeqNumbers.remove(sequenceNumber)) {
// late out-of-order delivery was detected
// decrease the loss counter
msgErrLossCounter.dec();
// increment the out-of-order counter
msgErrOutOfSeqCounter.inc();
} else {
msgErrDuplicateCounter.inc();
}
return;
}
boolean messagesSkipped = false;
if (sequenceNumber > expectedNumber) {
if (pendingOutOfSeqNumbers.size() == maxTrackOutOfOrderSequenceNumbers) {
messagesSkipped = processLowestPendingOutOfSequenceNumber();
}
if (!pendingOutOfSeqNumbers.add(sequenceNumber)) {
msgErrDuplicateCounter.inc();
}
} else {
// sequenceNumber == expectedNumber
expectedNumber++;
}
processPendingOutOfSequenceNumbers(messagesSkipped);
cleanUpTooFarBehindOutOfSequenceNumbers();
}
private boolean processLowestPendingOutOfSequenceNumber() {
// remove the lowest pending out of sequence number
Long lowestOutOfSeqNumber = pendingOutOfSeqNumbers.first();
pendingOutOfSeqNumbers.remove(lowestOutOfSeqNumber);
if (lowestOutOfSeqNumber > expectedNumber) {
// skip the expected number ahead to the number after the lowest sequence number
// increment the counter with the amount of sequence numbers that got skipped
// keep track of the skipped sequence numbers to detect late out-of-order message delivery
for (long l = expectedNumber; l < lowestOutOfSeqNumber; l++) {
msgErrLossCounter.inc();
skippedSeqNumbers.add(l);
if (skippedSeqNumbers.size() > maxTrackSkippedSequenceNumbers) {
skippedSeqNumbers.remove(skippedSeqNumbers.first());
}
}
expectedNumber = lowestOutOfSeqNumber + 1;
return true;
} else {
msgErrLossCounter.inc();
}
return false;
}
private void processPendingOutOfSequenceNumbers(boolean messagesSkipped) {
// check if there are previously received out-of-order sequence number that have been received
while (pendingOutOfSeqNumbers.remove(expectedNumber)) {
expectedNumber++;
if (!messagesSkipped) {
msgErrOutOfSeqCounter.inc();
}
}
}
private void cleanUpTooFarBehindOutOfSequenceNumbers() {
// remove sequence numbers that are too far behind
for (Iterator<Long> iterator = pendingOutOfSeqNumbers.iterator(); iterator.hasNext(); ) {
Long number = iterator.next();
if (number < expectedNumber - maxTrackOutOfOrderSequenceNumbers) {
msgErrLossCounter.inc();
iterator.remove();
} else {
break;
}
}
}
/**
* Handles the possible pending out of sequence numbers. Mainly needed in unit tests to assert the
* counter values.
*/
@Override
public void close() {
while (!pendingOutOfSeqNumbers.isEmpty()) {
processPendingOutOfSequenceNumbers(processLowestPendingOutOfSequenceNumber());
}
}
public int getMaxTrackOutOfOrderSequenceNumbers() {
return maxTrackOutOfOrderSequenceNumbers;
}
public int getMaxTrackSkippedSequenceNumbers() {
return maxTrackSkippedSequenceNumbers;
}
}

33
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/SyncPulsarOp.java
View File

@ -1,33 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
/**
* Base type of all Sync Pulsar Operations including Producers and Consumers.
*/
public abstract class SyncPulsarOp implements PulsarOp {
public void run(Runnable timeTracker) {
try {
this.run();
} finally {
timeTracker.run();
}
}
public abstract void run();
}

123
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/AvroUtil.java
View File

@ -1,123 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.util;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.JsonDecoder;
import org.apache.avro.io.BinaryDecoder;
import org.apache.pulsar.client.api.schema.Field;
import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.api.schema.GenericRecordBuilder;
import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
import org.apache.pulsar.common.schema.SchemaInfo;
import org.apache.pulsar.common.schema.SchemaType;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
public class AvroUtil {
////////////////////////
// Get an OSS Apache Avro schema from a string definition
public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) {
return new org.apache.avro.Schema.Parser().parse(avroSchemDef);
}
// Get an OSS Apache Avro schema record from a JSON string that matches a specific OSS Apache Avro schema
public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, String jsonData) {
org.apache.avro.generic.GenericRecord record = null;
try {
org.apache.avro.generic.GenericDatumReader<org.apache.avro.generic.GenericData.Record> reader;
reader = new org.apache.avro.generic.GenericDatumReader<>(schema);
JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, jsonData);
record = reader.read(null, decoder);
} catch (IOException ioe) {
ioe.printStackTrace();
}
return record;
}
// Get an OSS Apache Avro schema record from a byte array that matches a specific OSS Apache Avro schema
public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, byte[] bytesData) {
org.apache.avro.generic.GenericRecord record = null;
try {
org.apache.avro.generic.GenericDatumReader<org.apache.avro.generic.GenericData.Record> reader;
reader = new org.apache.avro.generic.GenericDatumReader<>(schema);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytesData, null);
record = reader.read(null, decoder);
} catch (IOException ioe) {
ioe.printStackTrace();
}
return record;
}
////////////////////////
// Get a Pulsar Avro schema from a string definition
public static GenericAvroSchema GetSchema_PulsarAvro(String schemaName, String avroSchemDef) {
SchemaInfo schemaInfo = SchemaInfoImpl.builder()
.schema(avroSchemDef.getBytes(StandardCharsets.UTF_8))
.type(SchemaType.AVRO)
.properties(new HashMap<>())
.name(schemaName)
.build();
return new GenericAvroSchema(schemaInfo);
}
// Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema
public static GenericRecord GetGenericRecord_PulsarAvro(
GenericAvroSchema pulsarGenericAvroSchema,
org.apache.avro.generic.GenericRecord apacheAvroGenericRecord)
{
GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder();
List<Field> fieldList = pulsarGenericAvroSchema.getFields();
for (Field field : fieldList) {
String fieldName = field.getName();
recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName));
}
return recordBuilder.build();
}
// Get a Pulsar Avro record (GenericRecord) from a JSON string that matches a specific Pulsar Avro schema
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, String avroSchemDefStr, String jsonData) {
org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr);
return GetGenericRecord_PulsarAvro(genericAvroSchema, avroSchema, jsonData);
}
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, org.apache.avro.Schema avroSchema, String jsonData) {
org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData);
return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord);
}
public static GenericRecord GetGenericRecord_PulsarAvro(String schemaName, String avroSchemDefStr, String jsonData) {
GenericAvroSchema genericAvroSchema = GetSchema_PulsarAvro(schemaName, avroSchemDefStr);
org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr);
org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData);
return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord);
}
}

535
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarActivityUtil.java
View File

@ -1,535 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.pulsar.client.api.Schema;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class PulsarActivityUtil {
private final static Logger logger = LogManager.getLogger(PulsarActivityUtil.class);
// Supported message operation types
// TODO: websocket-producer and managed-ledger
public enum OP_TYPES {
ADMIN_TENANT("admin-tenant"),
ADMIN_NAMESPACE("admin-namespace"),
ADMIN_TOPIC("admin-topic"),
E2E_MSG_PROC_SEND("e22-msg-proc-send"),
E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"),
// BATCH_MSG_SEND_START("batch-msg-send-start"),
// BATCH_MSG_SEND("batch-msg-send"),
// BATCH_MSG_SEND_END("batch-msg-send-end"),
MSG_SEND("msg-send"),
MSG_CONSUME("msg-consume"),
MSG_READ("msg-read"),
MSG_MULTI_CONSUME("msg-mt-consume");
public final String label;
OP_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidClientType(String type) {
return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
public static final String MSG_SEQUENCE_NUMBER = "sequence_number";
///////
// Valid document level parameters for Pulsar NB yaml file
public enum DOC_LEVEL_PARAMS {
TOPIC_URI("topic_uri"),
ASYNC_API("async_api"),
USE_TRANSACTION("use_transaction"),
ADMIN_DELOP("admin_delop"),
SEQ_TRACKING("seq_tracking"),
MSG_DEDUP_BROKER("msg_dedup_broker"),
E2E_STARTING_TIME_SOURCE("e2e_starting_time_source");
public final String label;
DOC_LEVEL_PARAMS(String label) {
this.label = label;
}
}
public static boolean isValidDocLevelParam(String param) {
return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param));
}
///////
// Valid Pulsar API type
public enum PULSAR_API_TYPE {
PRODUCER("producer"),
CONSUMER("consumer"),
READER("reader");
public final String label;
PULSAR_API_TYPE(String label) {
this.label = label;
}
}
public static boolean isValidPulsarApiType(String param) {
return Arrays.stream(PULSAR_API_TYPE.values()).anyMatch(t -> t.label.equals(param));
}
public static String getValidPulsarApiTypeList() {
return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
}
///////
// Valid persistence type
public enum PERSISTENT_TYPES {
PERSISTENT("persistent"),
NON_PERSISTENT("non-persistent")
;
public final String label;
PERSISTENT_TYPES(String label) {
this.label = label;
}
}
public static boolean isValidPersistenceType(String type) {
return Arrays.stream(PERSISTENT_TYPES.values()).anyMatch(t -> t.label.equals(type));
}
///////
// Valid Pulsar client configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#client
public enum CLNT_CONF_KEY {
serviceUrl("serviceUrl"),
authPulginClassName("authPluginClassName"),
authParams("authParams"),
pperationTimeoutMs("operationTimeoutMs"),
statsIntervalSeconds("statsIntervalSeconds"),
numIoThreads("numIoThreads"),
numListenerThreads("numListenerThreads"),
useTcpNoDelay("useTcpNoDelay"),
useTls("useTls"),
tlsTrustCertsFilePath("tlsTrustCertsFilePath"),
tlsAllowInsecureConnection("tlsAllowInsecureConnection"),
tlsHostnameVerificationEnable("tlsHostnameVerificationEnable"),
concurrentLookupRequest("concurrentLookupRequest"),
maxLookupRequest("maxLookupRequest"),
maxNumberOfRejectedRequestPerConnection("maxNumberOfRejectedRequestPerConnection"),
keepAliveIntervalSeconds("keepAliveIntervalSeconds"),
connectionTimeoutMs("connectionTimeoutMs"),
requestTimeoutMs("requestTimeoutMs"),
defaultBackoffIntervalNanos("defaultBackoffIntervalNanos"),
maxBackoffIntervalNanos("maxBackoffIntervalNanos")
;
public final String label;
CLNT_CONF_KEY(String label) {
this.label = label;
}
}
public static boolean isValidClientConfItem(String item) {
return Arrays.stream(CLNT_CONF_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Standard producer configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
public enum PRODUCER_CONF_STD_KEY {
topicName("topicName"),
producerName("producerName"),
sendTimeoutMs("sendTimeoutMs"),
blockIfQueueFull("blockIfQueueFull"),
maxPendingMessages("maxPendingMessages"),
maxPendingMessagesAcrossPartitions("maxPendingMessagesAcrossPartitions"),
messageRoutingMode("messageRoutingMode"),
hashingScheme("hashingScheme"),
cryptoFailureAction("cryptoFailureAction"),
batchingMaxPublishDelayMicros("batchingMaxPublishDelayMicros"),
batchingMaxMessages("batchingMaxMessages"),
batchingEnabled("batchingEnabled"),
compressionType("compressionType");
public final String label;
PRODUCER_CONF_STD_KEY(String label) {
this.label = label;
}
}
public static boolean isStandardProducerConfItem(String item) {
return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Standard consumer configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer
public enum CONSUMER_CONF_STD_KEY {
topicNames("topicNames"),
topicsPattern("topicsPattern"),
subscriptionName("subscriptionName"),
subscriptionType("subscriptionType"),
receiverQueueSize("receiverQueueSize"),
acknowledgementsGroupTimeMicros("acknowledgementsGroupTimeMicros"),
negativeAckRedeliveryDelayMicros("negativeAckRedeliveryDelayMicros"),
maxTotalReceiverQueueSizeAcrossPartitions("maxTotalReceiverQueueSizeAcrossPartitions"),
consumerName("consumerName"),
ackTimeoutMillis("ackTimeoutMillis"),
tickDurationMillis("tickDurationMillis"),
priorityLevel("priorityLevel"),
cryptoFailureAction("cryptoFailureAction"),
properties("properties"),
readCompacted("readCompacted"),
subscriptionInitialPosition("subscriptionInitialPosition"),
patternAutoDiscoveryPeriod("patternAutoDiscoveryPeriod"),
regexSubscriptionMode("regexSubscriptionMode"),
deadLetterPolicy("deadLetterPolicy"),
autoUpdatePartitions("autoUpdatePartitions"),
replicateSubscriptionState("replicateSubscriptionState");
public final String label;
CONSUMER_CONF_STD_KEY(String label) {
this.label = label;
}
}
public static boolean isStandardConsumerConfItem(String item) {
return Arrays.stream(CONSUMER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Custom consumer configuration (activity-level settings)
// - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer
// - NB Pulsar driver consumer operation specific
public enum CONSUMER_CONF_CUSTOM_KEY {
timeout("timeout");
public final String label;
CONSUMER_CONF_CUSTOM_KEY(String label) {
this.label = label;
}
}
public static boolean isCustomConsumerConfItem(String item) {
return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Pulsar subscription type
public enum SUBSCRIPTION_TYPE {
Exclusive("Exclusive"),
Failover("Failover"),
Shared("Shared"),
Key_Shared("Key_Shared");
public final String label;
SUBSCRIPTION_TYPE(String label) {
this.label = label;
}
}
public static boolean isValidSubscriptionType(String item) {
return Arrays.stream(SUBSCRIPTION_TYPE.values()).anyMatch(t -> t.label.equals(item));
}
public static String getValidSubscriptionTypeList() {
return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
}
///////
// Standard reader configuration (activity-level settings)
// - https://pulsar.apache.org/docs/en/client-libraries-java/#reader
public enum READER_CONF_STD_KEY {
topicName("topicName"),
receiverQueueSize("receiverQueueSize"),
readerListener("readerListener"),
readerName("readerName"),
subscriptionRolePrefix("subscriptionRolePrefix"),
cryptoKeyReader("cryptoKeyReader"),
cryptoFailureAction("cryptoFailureAction"),
readCompacted("readCompacted"),
resetIncludeHead("resetIncludeHead");
public final String label;
READER_CONF_STD_KEY(String label) {
this.label = label;
}
}
public static boolean isStandardReaderConfItem(String item) {
return Arrays.stream(READER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Custom reader configuration (activity-level settings)
// - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#reader
// - NB Pulsar driver reader operation specific
public enum READER_CONF_CUSTOM_KEY {
startMessagePos("startMessagePos");
public final String label;
READER_CONF_CUSTOM_KEY(String label) {
this.label = label;
}
}
public static boolean isCustomReaderConfItem(String item) {
return Arrays.stream(READER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Valid read positions for a Pulsar reader
public enum READER_MSG_POSITION_TYPE {
earliest("earliest"),
latest("latest"),
custom("custom");
public final String label;
READER_MSG_POSITION_TYPE(String label) {
this.label = label;
}
}
public static boolean isValideReaderStartPosition(String item) {
return Arrays.stream(READER_MSG_POSITION_TYPE.values()).anyMatch(t -> t.label.equals(item));
}
///////
// Pulsar subscription type
public enum SEQ_ERROR_SIMU_TYPE {
OutOfOrder("out_of_order"),
MsgLoss("msg_loss"),
MsgDup("msg_dup");
public final String label;
SEQ_ERROR_SIMU_TYPE(String label) {
this.label = label;
}
private static final Map<String, SEQ_ERROR_SIMU_TYPE> MAPPING = new HashMap<>();
static {
for (SEQ_ERROR_SIMU_TYPE simuType : values()) {
MAPPING.put(simuType.label, simuType);
MAPPING.put(simuType.label.toLowerCase(), simuType);
MAPPING.put(simuType.label.toUpperCase(), simuType);
MAPPING.put(simuType.name(), simuType);
MAPPING.put(simuType.name().toLowerCase(), simuType);
MAPPING.put(simuType.name().toUpperCase(), simuType);
}
}
public static Optional<SEQ_ERROR_SIMU_TYPE> parseSimuType(String simuTypeString) {
return Optional.ofNullable(MAPPING.get(simuTypeString.trim()));
}
}
public static boolean isValidSeqErrSimuType(String item) {
return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item));
}
public static String getValidSeqErrSimuTypeList() {
return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
}
///////
// Valid websocket-producer configuration (activity-level settings)
// TODO: to be added
public enum WEBSKT_PRODUCER_CONF_KEY {
;
public final String label;
WEBSKT_PRODUCER_CONF_KEY(String label) {
this.label = label;
}
}
///////
// Valid managed-ledger configuration (activity-level settings)
// TODO: to be added
public enum MANAGED_LEDGER_CONF_KEY {
;
public final String label;
MANAGED_LEDGER_CONF_KEY(String label) {
this.label = label;
}
}
///////
// Primitive Schema type
public static boolean isPrimitiveSchemaTypeStr(String typeStr) {
boolean isPrimitive = false;
// Use "BYTES" as the default type if the type string is not explicitly specified
if (StringUtils.isBlank(typeStr)) {
typeStr = "BYTES";
}
if (typeStr.equalsIgnoreCase("BOOLEAN") || typeStr.equalsIgnoreCase("INT8") ||
typeStr.equalsIgnoreCase("INT16") || typeStr.equalsIgnoreCase("INT32") ||
typeStr.equalsIgnoreCase("INT64") || typeStr.equalsIgnoreCase("FLOAT") ||
typeStr.equalsIgnoreCase("DOUBLE") || typeStr.equalsIgnoreCase("BYTES") ||
typeStr.equalsIgnoreCase("DATE") || typeStr.equalsIgnoreCase("TIME") ||
typeStr.equalsIgnoreCase("TIMESTAMP") || typeStr.equalsIgnoreCase("INSTANT") ||
typeStr.equalsIgnoreCase("LOCAL_DATE") || typeStr.equalsIgnoreCase("LOCAL_TIME") ||
typeStr.equalsIgnoreCase("LOCAL_DATE_TIME")) {
isPrimitive = true;
}
return isPrimitive;
}
public static Schema<?> getPrimitiveTypeSchema(String typeStr) {
Schema<?> schema;
switch (typeStr.toUpperCase()) {
case "BOOLEAN":
schema = Schema.BOOL;
break;
case "INT8":
schema = Schema.INT8;
break;
case "INT16":
schema = Schema.INT16;
break;
case "INT32":
schema = Schema.INT32;
break;
case "INT64":
schema = Schema.INT64;
break;
case "FLOAT":
schema = Schema.FLOAT;
break;
case "DOUBLE":
schema = Schema.DOUBLE;
break;
case "DATE":
schema = Schema.DATE;
break;
case "TIME":
schema = Schema.TIME;
break;
case "TIMESTAMP":
schema = Schema.TIMESTAMP;
break;
case "INSTANT":
schema = Schema.INSTANT;
break;
case "LOCAL_DATE":
schema = Schema.LOCAL_DATE;
break;
case "LOCAL_TIME":
schema = Schema.LOCAL_TIME;
break;
case "LOCAL_DATE_TIME":
schema = Schema.LOCAL_DATE_TIME;
break;
// Use BYTES as the default schema type if the type string is not specified
case "":
case "BYTES":
schema = Schema.BYTES;
break;
// Report an error if non-valid, non-empty schema type string is provided
default:
throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr);
}
return schema;
}
///////
// Complex strut type: Avro or Json
public static boolean isAvroSchemaTypeStr(String typeStr) {
return typeStr.equalsIgnoreCase("AVRO");
}
public static boolean isKeyValueTypeStr(String typeStr) {
return typeStr.equalsIgnoreCase("KEY_VALUE");
}
// automatic decode the type from the Registry
public static boolean isAutoConsumeSchemaTypeStr(String typeStr) {
return typeStr.equalsIgnoreCase("AUTO_CONSUME");
}
public static Schema<?> getAvroSchema(String typeStr, String definitionStr) {
String schemaDefinitionStr = definitionStr;
String filePrefix = "file://";
Schema<?> schema;
// Check if payloadStr points to a file (e.g. "file:///path/to/a/file")
if (isAvroSchemaTypeStr(typeStr)) {
if (StringUtils.isBlank(schemaDefinitionStr)) {
throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!");
} else if (schemaDefinitionStr.startsWith(filePrefix)) {
try {
Path filePath = Paths.get(URI.create(schemaDefinitionStr));
schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII);
} catch (IOException ioe) {
throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage());
}
}
schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr);
} else {
throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr);
}
return schema;
}
///////
// Generate effective key string
public static String buildCacheKey(String... keyParts) {
// Ignore blank keyPart
String joinedKeyStr =
Stream.of(keyParts)
.filter(s -> !StringUtils.isBlank(s))
.collect(Collectors.joining(","));
return Base64.getEncoder().encodeToString(joinedKeyStr.getBytes());
}
///////
// Convert JSON string to a key/value map
public static Map<String, String> convertJsonToMap(String jsonStr) throws Exception {
ObjectMapper mapper = new ObjectMapper();
return mapper.readValue(jsonStr, Map.class);
}
///////
// Get full namespace name (<tenant>/<namespace>) from a Pulsar topic URI
public static String getFullNamespaceName(String topicUri) {
// Get tenant/namespace string
// - topicUri : persistent://<tenant>/<namespace>/<topic>
// - tmpStr : <tenant>/<namespace>/<topic>
// - fullNsName : <tenant>/<namespace>
String tmpStr = StringUtils.substringAfter(topicUri,"://");
return StringUtils.substringBeforeLast(tmpStr, "/");
}
}

336
driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java
View File

@ -1,336 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.util;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.configuration2.FileBasedConfiguration;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
import org.apache.commons.configuration2.builder.fluent.Parameters;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class PulsarNBClientConf {
private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class);
private String canonicalFilePath = "";
public static final String SCHEMA_CONF_PREFIX = "schema";
public static final String CLIENT_CONF_PREFIX = "client";
public static final String PRODUCER_CONF_PREFIX = "producer";
public static final String CONSUMER_CONF_PREFIX = "consumer";
public static final String READER_CONF_PREFIX = "reader";
private final HashMap<String, Object> schemaConfMap = new HashMap<>();
private final HashMap<String, Object> clientConfMap = new HashMap<>();
private final HashMap<String, Object> producerConfMap = new HashMap<>();
private final HashMap<String, Object> consumerConfMap = new HashMap<>();
private final HashMap<String, Object> readerConfMap = new HashMap<>();
// TODO: add support for other operation types: websocket-producer, managed-ledger
public PulsarNBClientConf(String fileName) {
File file = new File(fileName);
try {
canonicalFilePath = file.getCanonicalPath();
Parameters params = new Parameters();
FileBasedConfigurationBuilder<FileBasedConfiguration> builder =
new FileBasedConfigurationBuilder<FileBasedConfiguration>(PropertiesConfiguration.class)
.configure(params.properties()
.setFileName(fileName));
Configuration config = builder.getConfiguration();
// Get schema specific configuration settings
for (Iterator<String> it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get client connection specific configuration settings
for (Iterator<String> it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get producer specific configuration settings
for (Iterator<String> it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get consumer specific configuration settings
for (Iterator<String> it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
// Get reader specific configuration settings
for (Iterator<String> it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) {
String confKey = it.next();
String confVal = config.getProperty(confKey).toString();
if (!StringUtils.isBlank(confVal))
readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
}
} catch (IOException ioe) {
logger.error("Can't read the specified config properties file!");
ioe.printStackTrace();
} catch (ConfigurationException cex) {
logger.error("Error loading configuration items from the specified config properties file: " + canonicalFilePath);
cex.printStackTrace();
}
}
//////////////////
// Get Schema related config
public Map<String, Object> getSchemaConfMap() {
return this.schemaConfMap;
}
public boolean hasSchemaConfKey(String key) {
if (key.contains(SCHEMA_CONF_PREFIX))
return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1));
else
return schemaConfMap.containsKey(key);
}
public Object getSchemaConfValue(String key) {
if (key.contains(SCHEMA_CONF_PREFIX))
return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1));
else
return schemaConfMap.get(key);
}
public void setSchemaConfValue(String key, Object value) {
if (key.contains(SCHEMA_CONF_PREFIX))
schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value);
else
schemaConfMap.put(key, value);
}
//////////////////
// Get Pulsar client related config
public Map<String, Object> getClientConfMap() {
return this.clientConfMap;
}
public boolean hasClientConfKey(String key) {
if (key.contains(CLIENT_CONF_PREFIX))
return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1));
else
return clientConfMap.containsKey(key);
}
public Object getClientConfValue(String key) {
if (key.contains(CLIENT_CONF_PREFIX))
return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1));
else
return clientConfMap.get(key);
}
public void setClientConfValue(String key, Object value) {
if (key.contains(CLIENT_CONF_PREFIX))
clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value);
else
clientConfMap.put(key, value);
}
//////////////////
// Get Pulsar producer related config
public Map<String, Object> getProducerConfMap() {
return this.producerConfMap;
}
public boolean hasProducerConfKey(String key) {
if (key.contains(PRODUCER_CONF_PREFIX))
return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1));
else
return producerConfMap.containsKey(key);
}
public Object getProducerConfValue(String key) {
if (key.contains(PRODUCER_CONF_PREFIX))
return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1));
else
return producerConfMap.get(key);
}
public void setProducerConfValue(String key, Object value) {
if (key.contains(PRODUCER_CONF_PREFIX))
producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value);
else
producerConfMap.put(key, value);
}
// other producer helper functions ...
public String getProducerName() {
Object confValue = getProducerConfValue(
"producer." + PulsarActivityUtil.PRODUCER_CONF_STD_KEY.producerName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getProducerTopicName() {
Object confValue = getProducerConfValue(
"producer." + PulsarActivityUtil.PRODUCER_CONF_STD_KEY.topicName);
if (confValue == null)
return "";
else
return confValue.toString();
}
//////////////////
// Get Pulsar consumer related config
public Map<String, Object> getConsumerConfMap() {
return this.consumerConfMap;
}
public boolean hasConsumerConfKey(String key) {
if (key.contains(CONSUMER_CONF_PREFIX))
return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1));
else
return consumerConfMap.containsKey(key);
}
public Object getConsumerConfValue(String key) {
if (key.contains(CONSUMER_CONF_PREFIX))
return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1));
else
return consumerConfMap.get(key);
}
public void setConsumerConfValue(String key, Object value) {
if (key.contains(CONSUMER_CONF_PREFIX))
consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value);
else
consumerConfMap.put(key, value);
}
// Other consumer helper functions ...
public String getConsumerTopicNames() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerTopicPattern() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerSubscriptionName() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerSubscriptionType() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getConsumerName() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
// NOTE: Below are not a standard Pulsar consumer configuration parameter as
// listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer"
// They're custom-made configuration properties for NB pulsar driver consumer.
public int getConsumerTimeoutSeconds() {
Object confValue = getConsumerConfValue(
"consumer." + PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
if (confValue == null)
return -1; // infinite
else
return Integer.parseInt(confValue.toString());
}
//////////////////
// Get Pulsar reader related config
public Map<String, Object> getReaderConfMap() {
return this.readerConfMap;
}
public boolean hasReaderConfKey(String key) {
if (key.contains(READER_CONF_PREFIX))
return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1));
else
return readerConfMap.containsKey(key);
}
public Object getReaderConfValue(String key) {
if (key.contains(READER_CONF_PREFIX))
return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1));
else
return readerConfMap.get(key);
}
public void setReaderConfValue(String key, Object value) {
if (key.contains(READER_CONF_PREFIX))
readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value);
else
readerConfMap.put(key, value);
}
// Other reader helper functions ...
public String getReaderTopicName() {
Object confValue = getReaderConfValue(
"reader." + PulsarActivityUtil.READER_CONF_STD_KEY.topicName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
public String getReaderName() {
Object confValue = getReaderConfValue(
"reader." + PulsarActivityUtil.READER_CONF_STD_KEY.readerName.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
// NOTE: Below are not a standard Pulsar reader configuration parameter as
// listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader"
// They're custom-made configuration properties for NB pulsar driver reader.
public String getStartMsgPosStr() {
Object confValue = getReaderConfValue(
"reader." + PulsarActivityUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label);
if (confValue == null)
return "";
else
return confValue.toString();
}
}

36
driver-pulsar/src/main/resources/activities/bindingtest.yaml
View File

@ -1,36 +0,0 @@
#
# Results:
# - 10 tenants
# - 2 namespaces per tanant
# - 5 topics per namespace
#------------------------------------------------------
#tenant=tenant_0 namespace=default_0 core_topic_name=t0
#tenant=tenant_0 namespace=default_0 core_topic_name=t1
#tenant=tenant_0 namespace=default_0 core_topic_name=t2
#tenant=tenant_0 namespace=default_0 core_topic_name=t3
#tenant=tenant_0 namespace=default_0 core_topic_name=t4
#tenant=tenant_0 namespace=default_1 core_topic_name=t0
#tenant=tenant_0 namespace=default_1 core_topic_name=t1
#tenant=tenant_0 namespace=default_1 core_topic_name=t2
#tenant=tenant_0 namespace=default_1 core_topic_name=t3
#tenant=tenant_0 namespace=default_1 core_topic_name=t4
#tenant=tenant_1 namespace=default_0 core_topic_name=t0
#tenant=tenant_1 namespace=default_0 core_topic_name=t1
#tenant=tenant_1 namespace=default_0 core_topic_name=t2
#tenant=tenant_1 namespace=default_0 core_topic_name=t3
#tenant=tenant_1 namespace=default_0 core_topic_name=t4
#tenant=tenant_1 namespace=default_1 core_topic_name=t0
#tenant=tenant_1 namespace=default_1 core_topic_name=t1
#tenant=tenant_1 namespace=default_1 core_topic_name=t2
#tenant=tenant_1 namespace=default_1 core_topic_name=t3
#tenant=tenant_1 namespace=default_1 core_topic_name=t4
# ... ...
bindings:
# message key and value
#mykey: NumberNameToString()
#myvalue: AlphaNumericString(20)
# Admin API - create tenant, namespace, and topic
tenant: Mod(100); Div(10L); ToString(); Prefix("tenant_")
namespace: Mod(10); Div(5L); ToString(); Prefix("default_")
core_topic_name: Mod(5); ToString(); Prefix("t")

52
driver-pulsar/src/main/resources/activities/config.properties
View File

@ -1,52 +0,0 @@
### Schema related configurations - schema.xxx
# valid types:
# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type)
# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue)
# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct)
# avro, json, protobuf
#
# TODO: as a starting point, only supports the following types
# 1) primitive types, including bytearray (byte[]) which is default, for messages without schema
# 2) Avro for messages with schema
#schema.type=avro
#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc
schema.type=
schema.definition=
### Pulsar client related configurations - client.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#client
client.connectionTimeoutMs=5000
client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken
# Cluster admin
client.authParams=
client.tlsAllowInsecureConnection=true
### Producer related configurations (global) - producer.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
producer.producerName=
producer.topicName=
producer.sendTimeoutMs=
producer.blockIfQueueFull=true
producer.maxPendingMessages=5000
producer.batchingMaxMessages=5000
### Consumer related configurations (global) - consumer.xxx
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer
consumer.topicNames=
consumer.topicsPattern=
consumer.subscriptionName=
consumer.subscriptionType=
consumer.consumerName=
consumer.receiverQueueSize=
### Reader related configurations (global) - reader.xxx
# https://pulsar.apache.org/docs/en/client-libraries-java/#reader
# - valid Pos: earliest, latest, custom::file://<path>/<to>/<message_id_file>
reader.topicName=
reader.receiverQueueSize=
reader.readerName=
reader.startMessagePos=earliest

11
driver-pulsar/src/main/resources/activities/iot-example.avsc
View File

@ -1,11 +0,0 @@
{
"type": "record",
"name": "IotSensor",
"namespace": "TestNS",
"fields" : [
{"name": "SensorID", "type": "string"},
{"name": "SensorType", "type": "string"},
{"name": "ReadingTime", "type": "string"},
{"name": "ReadingValue", "type": "float"}
]
}

22
driver-pulsar/src/main/resources/activities/pulsar_admin_namespace.yaml
View File

@ -1,22 +0,0 @@
bindings:
# 20 namespaces: 10 tenants, 2 namespaces/tenant
tenant: Mod(20); Div(2L); ToString(); Prefix("tnt")
namespace: Mod(2); ToString(); Prefix("ns")
params:
# "true" - asynchronous Pulsar Admin API
# "false" - synchronous Pulsar Admin API
async_api: "false"
# "true" - delete namespace
# "false" - create namespace
admin_delop: "false"
blocks:
- name: create-namespace-block
tags:
phase: admin-namespace
admin_task: true
statements:
- name: s1
optype: admin-namespace
namespace: "{tenant}/{namespace}"

23
driver-pulsar/src/main/resources/activities/pulsar_admin_tenant.yaml
View File

@ -1,23 +0,0 @@
bindings:
# 10 tenants
tenant: Mod(10); ToString(); Prefix("tnt")
params:
# "true" - asynchronous Pulsar Admin API
# "false" - synchronous Pulsar Admin API
async_api: "false"
# "true" - delete tenant
# "false" - create tenant
admin_delop: "true"
blocks:
- name: create-tenant-block
tags:
phase: admin-tenant
admin_task: true
statements:
- name: s1
optype: admin-tenant
# admin_roles:
# allowed_clusters:
tenant: "{tenant}"

25
driver-pulsar/src/main/resources/activities/pulsar_admin_topic.yaml
View File

@ -1,25 +0,0 @@
bindings:
# 100 topics: 10 tenants, 2 namespaces/tenant, 5 topics/namespace
tenant: Mod(100); Div(10L); ToString(); Prefix("tnt")
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
core_topic_name: Mod(5); ToString(); Prefix("t")
params:
topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
# "true" - asynchronous Pulsar Admin API
# "false" - synchronous Pulsar Admin API
async_api: "false"
# "true" - delete topic
# "false" - create topic
admin_delop: "false"
blocks:
- name: create-topic-block
tags:
phase: admin-topic
admin_task: true
statements:
- name: s1
optype: admin-topic
enable_partition: "false"
partition_num: "5"

93
driver-pulsar/src/main/resources/activities/pulsar_client_avro.yaml
View File

@ -1,93 +0,0 @@
bindings:
# message key and value
mykey: NumberNameToString()
sensor_id: ToUUID();ToString();
reading_time: ToDateTime();
reading_value: ToFloat(100);
tenant: Mod(100); Div(10L); ToString(); Prefix("tnt")
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
core_topic_name: Mod(5); ToString(); Prefix("t")
# document level parameters that apply to all Pulsar client types:
params:
topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
# topic_uri: "persistent://public/default/mytopic"
async_api: "true"
blocks:
- name: batch-producer-block
tags:
phase: batch-producer
admin_task: false
statements:
- name: s1
optype: batch-msg-send-start
# For batch producer, "producer_name" should be associated with batch start
# batch_producer_name: {batch_producer_name}
ratio: 1
- name: s2
optype: batch-msg-send
msg_key: "{mykey}"
msg_value: |
{
"SensorID": "{sensor_id}",
"SensorType": "Temperature",
"ReadingTime": "{reading_time}",
"ReadingValue": {reading_value}
}
ratio: 100
- name: s3
optype: batch-msg-send-end
ratio: 1
- name: producer-block
tags:
phase: producer
admin_task: false
statements:
- name: s1
optype: msg-send
# producer_name: {producer_name}
msg_key: "{mykey}"
msg_property: "{myprop}"
msg_value: |
{
"SensorID": "{sensor_id}",
"SensorType": "Temperature",
"ReadingTime": "{reading_time}",
"ReadingValue": {reading_value}
}
- name: consumer-block
tags:
phase: consumer
admin_task: false
statements:
- name: s1
optype: msg-consume
# topic_names:
# topics_pattern:
subscription_name: "mysub"
# subscription_type:
# consumer_name:
- name: reader-block
tags:
phase: reader
admin_task: false
statements:
- name: s1
optype: msg-read
# reader_name:
# - websocket-producer:
# tags:
# type: websocket-produer
# statements:
# - websocket-producer-stuff:
#
# - managed-ledger:
# tags:
# type: managed-ledger
# statement:
# - managed-ledger-stuff:

96
driver-pulsar/src/main/resources/activities/pulsar_client_kv.yaml
View File

@ -1,96 +0,0 @@
bindings:
# message key, property and value
mykey: NumberNameToString()
int_prop_val: ToString(); Prefix("IntProp_")
text_prop_val: AlphaNumericString(10); Prefix("TextProp_")
myvalue: NumberNameToString() #AlphaNumericString(20)
# tenant, namespace, and core topic name (without tenant and namespace)
tenant: Mod(100); Div(10L); ToString(); Prefix("tnt")
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
core_topic_name: Mod(5); ToString(); Prefix("t")
# document level parameters that apply to all Pulsar client types:
params:
topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
async_api: "true"
blocks:
- name: batch-producer-block
tags:
phase: batch-producer
admin_task: false
statements:
- name: s1
optype: batch-msg-send-start
# For batch producer, "producer_name" should be associated with batch start
# batch_producer_name: {batch_producer_name}
ratio: 1
- name: s2
optype: batch-msg-send
msg_key: "{mykey}"
msg_property: |
{
"prop1": "{int_prop_val}",
"prop2": "{text_prop_val}}"
}
msg_value: "{myvalue}"
ratio: 100
- name: s3
optype: batch-msg-send-end
ratio: 1
- name: producer-block
tags:
phase: producer
admin_task: false
statements:
- name: s1
optype: msg-send
# producer_name: {producer_name}
# msg_key:
msg_value: "{myvalue}"
- name: consumer-block
tags:
phase: consumer
admin_task: false
statements:
- name: s1
optype: msg-consume
subscription_name: "mysub"
# subscription_type:
# consumer_name:
- name: reader-block
tags:
phase: reader
admin_task: false
statements:
- name: s1
optype: msg-read
# reader_name:
- name: multi-topic-consumer-block
tags:
phase: multi-topic-consumer
admin_task: false
statements:
- name: s1
optype: msg-mt-consume
# topic_names:
# topics_pattern:
subscription_name: "mysub"
# subscription_type:
# consumer_name:
# - websocket-producer:
# tags:
# type: websocket-producer
# statements:
# - websocket-producer-stuff:
#
# - managed-ledger:
# tags:
# type: managed-ledger
# statement:
# - managed-ledger-stuff:

30
driver-pulsar/src/main/resources/activities/pulsar_client_sanity_e2e.yaml
View File

@ -1,30 +0,0 @@
bindings:
# message key, property and value
myprop1: AlphaNumericString(10); Prefix("PropVal_")
myvalue: NumberNameToString() #AlphaNumericString(20)
# document level parameters that apply to all Pulsar client types:
params:
topic_uri: "persistent://public/default/sanity_e2e_2"
async_api: "true"
blocks:
- name: e2e-msg-proc-block
tags:
phase: e2e-msg-proc
admin_task: false
statements:
- name: s1
optype: ec2-msg-proc-send
# msg_key:
msg_property: |
{
"prop1": "{myprop1}"
}
msg_value: "{myvalue}"
ratio: 1
- name: s2
optype: ec2-msg-proc-consume
ratio: 1
subscription_name: "mysub"
# subscription_type:

39
driver-pulsar/src/main/resources/activities/pulsar_client_sanity_seqloss.yaml
View File

@ -1,39 +0,0 @@
bindings:
# message key, property and value
myprop1: AlphaNumericString(10)
myvalue: NumberNameToString()
# document level parameters that apply to all Pulsar client types:
params:
topic_uri: "persistent://tnt0/ns0/sanity_seqloss12"
# Only applicable to producer and consumer
# - used for message ordering and message loss check
async_api: "true"
seq_tracking: "true"
blocks:
- name: producer-block
tags:
phase: producer
admin_task: false
statements:
- name: s1
optype: msg-send
#seqerr_simu: "out_of_order"
#seqerr_simu: "msg_loss"
#seqerr_simu: "msg_dup"
#seqerr_simu: "out_of_order, msg_loss"
# msg_key:
# msg_property:
msg_value: "{myvalue}"
- name: consumer-block
tags:
phase: consumer
admin_task: false
statements:
- name: s1
optype: msg-consume
subscription_name: "mysub"
subscription_type: "Shared"
# consumer_name:

89
driver-pulsar/src/main/resources/design_revisit.md
View File

@ -1,89 +0,0 @@
# TODO : Design Revisit -- Advanced Driver Features
**NOTE**: The following text is based on the original multi-layer API
caching design which is not fully implemented at the moment. We need to
revisit the original design at some point in order to achieve maximum
testing flexibility.
To summarize, the original caching design has the following key
requirements:
* **Requirement 1**: Each NB Pulsar activity is able to launch and cache
multiple **client spaces**
* **Requirement 2**: Each client space can launch and cache multiple
Pulsar operators of the same type (producer, consumer, etc.)
* **Requirement 3**: The size of each Pulsar operator specific cached
space can be configurable.
In the current implementation, only requirement 2 is implemented.
* For requirement 1, the current implementation only supports one client
space per NB Pulsar activity
* For requirement 3, the cache space size is not configurable (no limit at
the moment)
## Other Activity Parameters
- **maxcached** - A default value to be applied to `max_clients`,
`max_producers`, `max_consumers`.
- default: `max_cached=100`
- **max_clients** - Clients cache size. This is the number of client
instances which are allowed to be cached in the NoSQLBench client
runtime. The clients cache automatically maintains a cache of unique
client instances internally. default: _maxcached_
- **max_operators** - Producers/Consumers/Readers cache size (per client
instance). Limits the number of instances which are allowed to be cached
per client instance. default: _maxcached_
## API Caching
This driver is tailored around the multi-tenancy and topic naming scheme
that is part of Apache Pulsar. Specifically, you can create an arbitrary
number of client instances, producers (per client), and consumers (per
client) depending on your testing requirements.
Further, the topic URI is composed from the provided qualifiers of
`persistence`, `tenant`, `namespace`, and `topic`, or you can provide a
fully-composed value in the `persistence://tenant/namespace/topic`
form.
### Instancing Controls
Normative usage of the Apache Pulsar API follows a strictly enforced
binding of topics to producers and consumers. As well, clients may be
customized with different behavior for advanced testing scenarios. There
is a significant variety of messaging and concurrency schemes seen in
modern architectures. Thus, it is important that testing tools rise to the
occasion by letting users configure their testing runtimes to emulate
applications as they are found in practice. To this end, the NoSQLBench
driver for Apache Pulsar provides a set controls within its op template
format which allow for flexible yet intuitive instancing in the client
runtime. This is enabled directly by using nominative variables for
instance names where needed. When the instance names are not provided for
an operation, defaults are used to emulate a simple configuration.
Since this is a new capability in a NoSQLBench driver, how it works is
explained below:
When a pulsar cycles is executed, the operation is synthesized from the op
template fields as explained below under _Op Fields_. This happens in a
specific order:
1. The client instance name is resolved. If a `client` field is provided,
this is taken as the client instance name. If not, it is set
to `default`.
2. The named client instance is fetched from the cache, or created and
cached if it does not yet exist.
3. The topic_uri is resolved. This is the value to be used with
`.topic(...)` calls in the API. The op fields below explain how to
control this value.
4. For _send_ operations, a producer is named and created if needed. By
default, the producer is named after the topic_uri above. You can
override this by providing a value for `producer`.
5. For _recv_ operations, a consumer is named and created if needed. By
default, the consumer is named after the topic_uri above. You can
override this by providing a value for `consumer`.
The most important detail for understanding the instancing controls is
that clients, producers, and consumers are all named and cached in the
specific order above.

34
driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/TestYamlLoader.java
View File

@ -1,34 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
public class TestYamlLoader {
private final static Logger logger = LogManager.getLogger(TestYamlLoader.class);
@Test
public void loadAvroYaml() {
RawStmtsLoader sl = new RawStmtsLoader();
RawStmtsDocList rsdl = sl.loadPath(logger, "activities/pulsar_client_avro.yaml");
}
}

90
driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandlerTest.java
View File

@ -1,90 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.junit.jupiter.api.Test;
class MessageSequenceNumberSendingHandlerTest {
MessageSequenceNumberSendingHandler sequenceNumberSendingHandler = new MessageSequenceNumberSendingHandler();
@Test
void shouldAddMonotonicSequence() {
for (long l = 1; l <= 100; l++) {
assertEquals(l, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
}
}
@Test
void shouldInjectMessageLoss() {
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
assertEquals(3L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.MsgLoss), 100));
}
@Test
void shouldInjectMessageDuplication() {
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.MsgDup), 100));
}
@Test
void shouldInjectMessageOutOfOrder() {
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
assertEquals(4L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.OutOfOrder), 100));
assertEquals(2L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
assertEquals(3L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
assertEquals(5L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
assertEquals(6, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
}
@Test
void shouldInjectOneOfTheSimulatedErrorsRandomly() {
Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> allErrorTypes = new HashSet<>(Arrays.asList(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.values()));
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
long previousSequenceNumber = 1L;
int outOfSequenceInjectionCounter = 0;
int messageDupCounter = 0;
int messageLossCounter = 0;
int successCounter = 0;
for (int i = 0; i < 1000; i++) {
long nextSequenceNumber = sequenceNumberSendingHandler.getNextSequenceNumber(allErrorTypes);
if (nextSequenceNumber >= previousSequenceNumber + 3) {
outOfSequenceInjectionCounter++;
} else if (nextSequenceNumber <= previousSequenceNumber) {
messageDupCounter++;
} else if (nextSequenceNumber >= previousSequenceNumber + 2) {
messageLossCounter++;
} else if (nextSequenceNumber == previousSequenceNumber + 1) {
successCounter++;
}
previousSequenceNumber = nextSequenceNumber;
}
assertTrue(outOfSequenceInjectionCounter > 0);
assertTrue(messageDupCounter > 0);
assertTrue(messageLossCounter > 0);
assertEquals(1000, outOfSequenceInjectionCounter + messageDupCounter + messageLossCounter + successCounter);
}
}

246
driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTrackerTest.java
View File

@ -1,246 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.driver.pulsar.ops;
import static org.junit.jupiter.api.Assertions.assertEquals;
import com.codahale.metrics.Counter;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
class ReceivedMessageSequenceTrackerTest {
Counter msgErrOutOfSeqCounter = new Counter();
Counter msgErrDuplicateCounter = new Counter();
Counter msgErrLossCounter = new Counter();
ReceivedMessageSequenceTracker messageSequenceTracker = new ReceivedMessageSequenceTracker(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter, 20, 20);
@Test
void shouldCountersBeZeroWhenSequenceDoesntContainGaps() {
// when
for (long l = 0; l < 100L; l++) {
messageSequenceTracker.sequenceNumberReceived(l);
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(0, msgErrLossCounter.getCount());
}
@ParameterizedTest
@ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L})
void shouldDetectMsgLossWhenEverySecondMessageIsLost(long totalMessages) {
doShouldDetectMsgLoss(totalMessages, 2);
}
@ParameterizedTest
@ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L})
void shouldDetectMsgLossWhenEveryThirdMessageIsLost(long totalMessages) {
doShouldDetectMsgLoss(totalMessages, 3);
}
@ParameterizedTest
@ValueSource(longs = {20L, 21L, 40L, 41L, 42L, 43L, 100L})
void shouldDetectMsgLossWhenEvery21stMessageIsLost(long totalMessages) {
doShouldDetectMsgLoss(totalMessages, 21);
}
private void doShouldDetectMsgLoss(long totalMessages, int looseEveryNthMessage) {
int messagesLost = 0;
// when
boolean lastMessageWasLost = false;
for (long l = 0; l < totalMessages; l++) {
if (l % looseEveryNthMessage == 1) {
messagesLost++;
lastMessageWasLost = true;
continue;
} else {
lastMessageWasLost = false;
}
messageSequenceTracker.sequenceNumberReceived(l);
}
if (lastMessageWasLost) {
messageSequenceTracker.sequenceNumberReceived(totalMessages);
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(messagesLost, msgErrLossCounter.getCount());
}
@ParameterizedTest
@ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L})
void shouldDetectMsgDuplication(long totalMessages) {
int messagesDuplicated = 0;
// when
for (long l = 0; l < totalMessages; l++) {
if (l % 2 == 1) {
messagesDuplicated++;
messageSequenceTracker.sequenceNumberReceived(l);
}
messageSequenceTracker.sequenceNumberReceived(l);
}
if (totalMessages % 2 == 0) {
messageSequenceTracker.sequenceNumberReceived(totalMessages);
}
if (totalMessages < 2 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers()) {
messageSequenceTracker.close();
}
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(messagesDuplicated, msgErrDuplicateCounter.getCount());
assertEquals(0, msgErrLossCounter.getCount());
}
@Test
void shouldDetectSingleMessageOutOfSequence() {
// when
for (long l = 0; l < 10L; l++) {
messageSequenceTracker.sequenceNumberReceived(l);
}
messageSequenceTracker.sequenceNumberReceived(10L);
messageSequenceTracker.sequenceNumberReceived(12L);
messageSequenceTracker.sequenceNumberReceived(11L);
for (long l = 13L; l < 100L; l++) {
messageSequenceTracker.sequenceNumberReceived(l);
}
// then
assertEquals(1, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(0, msgErrLossCounter.getCount());
}
@Test
void shouldDetectMultipleMessagesOutOfSequence() {
// when
for (long l = 0; l < 10L; l++) {
messageSequenceTracker.sequenceNumberReceived(l);
}
messageSequenceTracker.sequenceNumberReceived(10L);
messageSequenceTracker.sequenceNumberReceived(14L);
messageSequenceTracker.sequenceNumberReceived(13L);
messageSequenceTracker.sequenceNumberReceived(11L);
messageSequenceTracker.sequenceNumberReceived(12L);
for (long l = 15L; l < 100L; l++) {
messageSequenceTracker.sequenceNumberReceived(l);
}
// then
assertEquals(2, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(0, msgErrLossCounter.getCount());
}
@Test
void shouldDetectIndividualMessageLoss() {
// when
for (long l = 0; l < 100L; l++) {
if (l != 11L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(1, msgErrLossCounter.getCount());
}
@Test
void shouldDetectGapAndMessageDuplication() {
// when
for (long l = 0; l < 100L; l++) {
if (l != 11L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
if (l == 12L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(1, msgErrDuplicateCounter.getCount());
assertEquals(1, msgErrLossCounter.getCount());
}
@Test
void shouldDetectGapAndMessageDuplicationTimes2() {
// when
for (long l = 0; l < 100L; l++) {
if (l != 11L) {
messageSequenceTracker.sequenceNumberReceived(l);
}
if (l == 12L) {
messageSequenceTracker.sequenceNumberReceived(l);
messageSequenceTracker.sequenceNumberReceived(l);
}
}
messageSequenceTracker.close();
// then
assertEquals(0, msgErrOutOfSeqCounter.getCount());
assertEquals(2, msgErrDuplicateCounter.getCount());
assertEquals(1, msgErrLossCounter.getCount());
}
@Test
void shouldDetectDelayedOutOfOrderDelivery() {
// when
for (long l = 0; l < 5 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers(); l++) {
if (l != 10) {
messageSequenceTracker.sequenceNumberReceived(l);
}
if (l == messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers() * 2) {
messageSequenceTracker.sequenceNumberReceived(10);
}
}
messageSequenceTracker.close();
// then
assertEquals(1, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(0, msgErrLossCounter.getCount());
}
@Test
void shouldDetectDelayedOutOfOrderDeliveryOf2ConsecutiveSequenceNumbers() {
// when
for (long l = 0; l < 5 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers(); l++) {
if (l != 10 && l != 11) {
messageSequenceTracker.sequenceNumberReceived(l);
}
if (l == messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers() * 2) {
messageSequenceTracker.sequenceNumberReceived(10);
messageSequenceTracker.sequenceNumberReceived(11);
}
}
messageSequenceTracker.close();
// then
assertEquals(2, msgErrOutOfSeqCounter.getCount());
assertEquals(0, msgErrDuplicateCounter.getCount());
assertEquals(0, msgErrLossCounter.getCount());
}
}

6
nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java
View File

@ -22,9 +22,11 @@ public class BundledMarkdownLoader {
public static DocsBinder loadBundledMarkdown() { public static DocsBinder loadBundledMarkdown() {
ServiceLoader<BundledMarkdownManifest> loader = ServiceLoader.load(BundledMarkdownManifest.class); ServiceLoader<BundledMarkdownManifest> loader = ServiceLoader.load(BundledMarkdownManifest.class);
Docs docs = new Docs(); DocsBinder docs = new Docs();
for (BundledMarkdownManifest docPathInfos : loader) { for (BundledMarkdownManifest docPathInfos : loader) {
docs.merge(docPathInfos.getDocs());
DocsBinder docsBinder = docPathInfos.getDocs();
docs = docs.merge(docsBinder);
} }
return docs; return docs;

20
nb-api/src/main/java/io/nosqlbench/api/markdown/aggregator/MutableMarkdown.java
View File

@ -51,6 +51,11 @@ public class MutableMarkdown {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public MutableMarkdown(String rawMarkdown) {
this.path = null;
this.rawMarkdown = rawMarkdown;
parseStructure(rawMarkdown);
}
private void parseStructure(String rawMarkdown) { private void parseStructure(String rawMarkdown) {
AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor(); AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor();
@ -69,13 +74,19 @@ public class MutableMarkdown {
} else if (node instanceof WhiteSpace) { } else if (node instanceof WhiteSpace) {
} else if (node instanceof YamlFrontMatterBlock) { } else if (node instanceof YamlFrontMatterBlock) {
} else { } else {
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName()); if(this.path != null)
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName());
else
throw new RuntimeException("The markdown string provided must have an initial heading as a title, before any other element, but found: "+ node.getClass().getSimpleName());
} }
node=node.getNext(); node=node.getNext();
} }
} }
if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) { if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) {
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title."); if(this.path != null)
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title.");
else
throw new RuntimeException("The markdown string provided has no heading to use as a title.");
} }
} }
@ -90,7 +101,10 @@ public class MutableMarkdown {
if (end>=0) { if (end>=0) {
return rawMarkdown.substring(end+4); return rawMarkdown.substring(end+4);
} else { } else {
throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary); if(path != null)
throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary);
else
throw new RuntimeException("Unable to find matching boundaries in provided markdown: " + boundary);
} }
} }
} }

1
nbr/pom.xml
View File

@ -72,6 +72,7 @@
<artifactId>adapter-diag</artifactId> <artifactId>adapter-diag</artifactId>
<version>4.17.32-SNAPSHOT</version> <version>4.17.32-SNAPSHOT</version>
</dependency> </dependency>
</dependencies> </dependencies>
<build> <build>

2
nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java → nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java
View File

@ -39,7 +39,7 @@ public class BundledFrontmatterInjector implements BundledMarkdownProcessor {
if (name.length()>i) { if (name.length()>i) {
int ord = name.charAt(i) - 'a'; int ord = name.charAt(i) - 'a';
double addend = Math.pow(pow, i) * ord; double addend = Math.pow(pow, i) * ord;
sum += addend; sum += (int)addend;
} else { } else {
break; break;
} }

1
nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java → nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java
View File

@ -53,6 +53,7 @@ public class BundledMarkdownExporter implements BundledApp {
String zipfile = options.valueOf(zipfileSpec); String zipfile = options.valueOf(zipfileSpec);
new BundledMarkdownZipExporter(new BundledFrontmatterInjector()).exportDocs(Path.of(zipfile)); new BundledMarkdownZipExporter(new BundledFrontmatterInjector()).exportDocs(Path.of(zipfile));
return 0; return 0;
} }
} }

0
nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownProcessor.java → nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownProcessor.java
View File

38
nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java → nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java
View File

@ -20,6 +20,7 @@ import io.nosqlbench.api.docsapi.BundledMarkdownLoader;
import io.nosqlbench.api.docsapi.DocsBinder; import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.api.docsapi.DocsNameSpace; import io.nosqlbench.api.docsapi.DocsNameSpace;
import io.nosqlbench.api.markdown.aggregator.MutableMarkdown; import io.nosqlbench.api.markdown.aggregator.MutableMarkdown;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.VirtDataGenDocsApp;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -29,7 +30,12 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.util.Date;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Function; import java.util.function.Function;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
@ -51,12 +57,33 @@ public class BundledMarkdownZipExporter {
zipstream.setMethod(ZipOutputStream.DEFLATED); zipstream.setMethod(ZipOutputStream.DEFLATED);
zipstream.setLevel(9); zipstream.setLevel(9);
DocsBinder docsNameSpaces = BundledMarkdownLoader.loadBundledMarkdown(); DocsBinder docsNameSpaces = BundledMarkdownLoader.loadBundledMarkdown(); //Loads the drivers under @Service Annotation
for (DocsNameSpace docs_ns : docsNameSpaces) { for (DocsNameSpace docs_ns : docsNameSpaces) {
for (Path p : docs_ns) { for (Path p : docs_ns) {
addEntry(p, p.getParent(), zipstream); addEntry(p, p.getParent(), zipstream, docs_ns.getName() + "/");
} }
} }
ExecutorService executorService = Executors.newSingleThreadExecutor();
Future<Map<String, StringBuilder>> future = executorService.submit(new VirtDataGenDocsApp(null));
Map<String, StringBuilder> builderMap = future.get();
executorService.shutdown();
String bindingsPrefix ="bindings/";
for(Map.Entry<String, StringBuilder> entry : builderMap.entrySet())
{
String filename = entry.getKey();
StringBuilder fileStringBuilder = entry.getValue();
MutableMarkdown parsed = new MutableMarkdown(fileStringBuilder.toString());
for (BundledMarkdownProcessor filter : this.filters) {
parsed = filter.apply(parsed);
}
ZipEntry zipEntry = new ZipEntry(bindingsPrefix + filename);
zipEntry.setTime(new Date().getTime());
zipstream.putNextEntry(zipEntry);
zipstream.write(parsed.getComposedMarkdown().getBytes(StandardCharsets.UTF_8));
zipstream.closeEntry();
}
zipstream.finish(); zipstream.finish();
stream.close(); stream.close();
} catch (Exception e) { } catch (Exception e) {
@ -64,18 +91,17 @@ public class BundledMarkdownZipExporter {
} }
} }
private void addEntry(Path p, Path r, ZipOutputStream zos) throws IOException { private void addEntry(Path p, Path r, ZipOutputStream zos, String prefix) throws IOException {
String name = r.relativize(p).toString(); String name = r.relativize(p).toString();
name = Files.isDirectory(p) ? (name.endsWith(File.separator) ? name : name + File.separator) : name; name = Files.isDirectory(p) ? (name.endsWith(File.separator) ? name : name + File.separator) : name;
ZipEntry entry = new ZipEntry(prefix + name);
ZipEntry entry = new ZipEntry(name);
if (Files.isDirectory(p)) { if (Files.isDirectory(p)) {
zos.putNextEntry(entry); zos.putNextEntry(entry);
DirectoryStream<Path> stream = Files.newDirectoryStream(p); DirectoryStream<Path> stream = Files.newDirectoryStream(p);
for (Path path : stream) { for (Path path : stream) {
addEntry(path,r,zos); addEntry(path,r,zos, prefix);
} }
} else { } else {
entry.setTime(Files.getLastModifiedTime(p).toMillis()); entry.setTime(Files.getLastModifiedTime(p).toMillis());

0
nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java → nbr/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java
View File

0
nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownTestManifest.java → nbr/src/test/java/io/nosqlbench/docapi/BundledMarkdownTestManifest.java
View File

3
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_string/EscapeJSON.java
View File

@ -18,6 +18,8 @@ package io.nosqlbench.virtdata.library.basics.shared.from_string;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.google.gson.GsonBuilder; import com.google.gson.GsonBuilder;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.text.StringEscapeUtils;
@ -38,6 +40,7 @@ import java.util.function.Function;
* }</pre> * }</pre>
*/ */
@ThreadSafeMapper @ThreadSafeMapper
@Categories({Category.conversion, Category.general})
public class EscapeJSON implements Function<String,String> { public class EscapeJSON implements Function<String,String> {
Gson gson = new GsonBuilder().create(); Gson gson = new GsonBuilder().create();

9
virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java
View File

@ -31,12 +31,11 @@ import java.util.Arrays;
public class VirtDataMainApp implements BundledApp { public class VirtDataMainApp implements BundledApp {
private final static String APP_TESTMAPPER = "testmapper"; private final static String APP_TESTMAPPER = "testmapper";
private final static String APP_GENDOCS = "gendocs";
private final static String APP_DIAGNOSE = "diagnose"; private final static String APP_DIAGNOSE = "diagnose";
private final static String[] names = new String[]{APP_GENDOCS, APP_TESTMAPPER, APP_DIAGNOSE}; private final static String[] names = new String[]{APP_TESTMAPPER, APP_DIAGNOSE};
public static boolean hasNamedApp(String appname) { public static boolean hasNamedApp(String appname) {
return (appname.equals(APP_TESTMAPPER) || appname.equals(APP_GENDOCS) || appname.equals(APP_DIAGNOSE)); return (appname.equals(APP_TESTMAPPER) || appname.equals(APP_DIAGNOSE));
} }
public static void main(String[] args) { public static void main(String[] args) {
@ -46,7 +45,7 @@ public class VirtDataMainApp implements BundledApp {
@Override @Override
public int applyAsInt(String[] args) { public int applyAsInt(String[] args) {
if (args.length == 0) { if (args.length == 0) {
System.out.println("Usage: app (" + APP_TESTMAPPER + "|" + APP_GENDOCS + "|" + APP_DIAGNOSE +")"); System.out.println("Usage: app (" + APP_TESTMAPPER +"|"+ APP_DIAGNOSE +")");
return 1; return 1;
} }
@ -58,8 +57,6 @@ public class VirtDataMainApp implements BundledApp {
if (appSelection.equalsIgnoreCase(APP_TESTMAPPER)) { if (appSelection.equalsIgnoreCase(APP_TESTMAPPER)) {
VirtDataCheckPerfApp.main(appArgs); VirtDataCheckPerfApp.main(appArgs);
} else if (appSelection.equalsIgnoreCase(APP_GENDOCS)) {
VirtDataGenDocsApp.main(appArgs);
} else if (appSelection.equalsIgnoreCase(APP_DIAGNOSE)) { } else if (appSelection.equalsIgnoreCase(APP_DIAGNOSE)) {
VirtDataDiagnoseApp.main(appArgs); VirtDataDiagnoseApp.main(appArgs);
} else { } else {

50
virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/VirtDataGenDocsApp.java
View File

@ -36,8 +36,9 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.*; import java.util.*;
import java.util.concurrent.Callable;
public class VirtDataGenDocsApp implements Runnable { public class VirtDataGenDocsApp implements Callable<Map<String, StringBuilder>> {
private final static Logger logger = LogManager.getLogger(VirtDataGenDocsApp.class); private final static Logger logger = LogManager.getLogger(VirtDataGenDocsApp.class);
@ -55,6 +56,7 @@ public class VirtDataGenDocsApp implements Runnable {
private final String[] args; private final String[] args;
private final Map<String, Writer> writers = new HashMap<>(); private final Map<String, Writer> writers = new HashMap<>();
Map<String, StringBuilder> stringBuilders = new HashMap<String, StringBuilder>();
private String baseFileName = BASE_FILENAME; private String baseFileName = BASE_FILENAME;
private String categories = CATEGORIES_SPLIT; private String categories = CATEGORIES_SPLIT;
@ -64,22 +66,21 @@ public class VirtDataGenDocsApp implements Runnable {
private String basedir = ""; private String basedir = "";
public static void main(String[] args) { public static void main(String[] args) {
new VirtDataGenDocsApp(args).run(); new VirtDataGenDocsApp(args).call();
} }
public VirtDataGenDocsApp(String[] args) { public VirtDataGenDocsApp(String[] args) {this.args = args;}
this.args = args;
}
public void run() { public Map<String, StringBuilder> call() {
LinkedList<String> largs = new LinkedList<>(Arrays.asList(args));
/*LinkedList<String> largs = new LinkedList<>(Arrays.asList(args));
if (args.length > 0 && args[0].contains("help")) { if (args.length > 0 && args[0].contains("help")) {
System.out.println( System.out.println(
"usage:\n" + "usage:\n" +
"[basefile <name>] [basedir <dir>] [categories combined|split] [format json|markdown] " + "[basefile <name>] [basedir <dir>] [categories combined|split] [format json|markdown] " +
"[blurbsdirs <dir>[:...]]\n\n" "[blurbsdirs <dir>[:...]]\n\n"
); );
return; return result;
} }
while (largs.peekFirst() != null) { while (largs.peekFirst() != null) {
String argtype = largs.removeFirst(); String argtype = largs.removeFirst();
@ -112,12 +113,12 @@ public class VirtDataGenDocsApp implements Runnable {
break; break;
default: default:
} }
} }*/
Optional<FDoc> docsinfo = loadAllDocs(); Optional<FDoc> docsinfo = loadAllDocs();
if (!docsinfo.isPresent()) { if (!docsinfo.isPresent()) {
return; return stringBuilders;
} }
try { try {
@ -131,38 +132,43 @@ public class VirtDataGenDocsApp implements Runnable {
+ (this.categories.equals(CATEGORIES_SPLIT) ? "_" + categoryName : "") + (this.categories.equals(CATEGORIES_SPLIT) ? "_" + categoryName : "")
+ extension; + extension;
Writer writer = getWriterFor(filename); // Writer writer = getWriterFor(filename);
StringBuilder builder = getBuilderFor(filename);
String name = filename;
for (FDocFuncs docsForFuncName : docsForCatName) { for (FDocFuncs docsForFuncName : docsForCatName) {
if (format.equals(FORMAT_JSON)) { if (format.equals(FORMAT_JSON)) {
Gson gson = new GsonBuilder().setPrettyPrinting().create(); Gson gson = new GsonBuilder().setPrettyPrinting().create();
writer.append(gson.toJson(docsForFuncName)); builder.append(gson.toJson(docsForFuncName));
} else if (format.equals(FORMAT_MARKDOWN)) { } else if (format.equals(FORMAT_MARKDOWN)) {
String markdown = docsForFuncName.asMarkdown(); String markdown = docsForFuncName.asMarkdown();
writer.append(markdown); builder.append(markdown);
} }
} }
} }
for (Writer writer : writers.values()) { /*for (Writer writer : writers.values()) {
writer.flush(); writer.flush();
writer.close(); writer.close();
} }*/
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
return stringBuilders;
} }
private Writer getWriterFor(String outputname) { private StringBuilder getBuilderFor(String outputname) {
FileWriter fileWriter = null; // FileWriter fileWriter = null;
if (!writers.containsKey(outputname)) { StringBuilder builder = null;
if (!stringBuilders.containsKey(outputname)) {
try { try {
outputname = basedir.isEmpty() ? outputname : basedir + "/" + outputname; outputname = basedir.isEmpty() ? outputname : basedir + "/" + outputname;
Path parent = Path.of(outputname).getParent(); Path parent = Path.of(outputname).getParent();
if (parent != null) { if (parent != null) {
Files.createDirectories(parent); Files.createDirectories(parent);
} }
fileWriter = new FileWriter(outputname, false); builder = new StringBuilder();
writers.put(outputname, fileWriter); stringBuilders.put(outputname, builder);
String[] blurbsdirs = blurbsDirs.split(":"); String[] blurbsdirs = blurbsDirs.split(":");
for (String blurbsdir : blurbsdirs) { for (String blurbsdir : blurbsdirs) {
@ -173,7 +179,7 @@ public class VirtDataGenDocsApp implements Runnable {
String blurb = Files.readString(blurbsFile, StandardCharsets.UTF_8); String blurb = Files.readString(blurbsFile, StandardCharsets.UTF_8);
logger.debug("writing blurb to " + outputname); logger.debug("writing blurb to " + outputname);
fileWriter.append(blurb); builder.append(blurb);
} }
} }
} }
@ -181,7 +187,7 @@ public class VirtDataGenDocsApp implements Runnable {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
return writers.get(outputname); return stringBuilders.get(outputname);
} }
private Optional<FDoc> loadAllDocs() { private Optional<FDoc> loadAllDocs() {