mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
Merge pull request #863 from nosqlbench/my-NB5-I446
nb5 - Issue 446 - Modified the export-docs command and 'build' workflow
This commit is contained in:
commit
c4415e6c0c
118
.github/workflows/build.yml
vendored
118
.github/workflows/build.yml
vendored
@ -2,43 +2,105 @@ name: build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
name: checkout nosqlbench
|
||||
- uses: actions/checkout@v3
|
||||
name: checkout nosqlbench
|
||||
- uses: actions/setup-java@v3
|
||||
name: setup java
|
||||
with:
|
||||
java-version: '17'
|
||||
java-package: jdk
|
||||
architecture: x64
|
||||
distribution: 'temurin'
|
||||
|
||||
- uses: actions/setup-java@v3
|
||||
name: setup java
|
||||
with:
|
||||
java-version: '17'
|
||||
java-package: jdk
|
||||
architecture: x64
|
||||
distribution: 'temurin'
|
||||
- name: Cache Maven packages
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: ${{ runner.os }}-m2
|
||||
|
||||
- name: Cache Maven packages
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: ${{ runner.os }}-m2
|
||||
- name: mvn-package
|
||||
run: mvn package
|
||||
|
||||
- name: mvn-package
|
||||
run: mvn package
|
||||
- name: export docs
|
||||
run: nb5/target/nb5 export-docs
|
||||
|
||||
- name: mvn-verify
|
||||
run: mvn verify
|
||||
- name: upload docs artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: exported-docs
|
||||
path: exported_docs.zip
|
||||
|
||||
- name: Capture
|
||||
if: success() || failure()
|
||||
run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
|
||||
- name: mvn verify
|
||||
run: mvn verify
|
||||
|
||||
- name: Archive Test Results
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: test-results
|
||||
path: logfiles.tar
|
||||
- name: Capture
|
||||
if: success() || failure()
|
||||
run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
|
||||
|
||||
- name: Archive Test Results
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: test-results
|
||||
path: logfiles.tar
|
||||
|
||||
|
||||
docs:
|
||||
needs: build
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: set git username
|
||||
run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
|
||||
|
||||
- name: set git email
|
||||
run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
|
||||
|
||||
- name: download exported-docs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: exported-docs
|
||||
|
||||
- name: unzip docs
|
||||
run: unzip exported_docs.zip
|
||||
|
||||
- run: ls -la
|
||||
|
||||
- name: clone nosqlbench-build-docs
|
||||
env:
|
||||
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
|
||||
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
|
||||
run: |
|
||||
git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs
|
||||
cd nosqlbench-build-docs
|
||||
echo "files listing"
|
||||
find .
|
||||
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git
|
||||
git remote -v
|
||||
|
||||
- name: push changes
|
||||
env:
|
||||
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
|
||||
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
|
||||
run: |
|
||||
rsync -av --delete -I --exclude '_index.md' drivers/ nosqlbench-build-docs/site/content/docs/drivers
|
||||
rsync -av --delete -I --exclude '_index.md' bindings/ nosqlbench-build-docs/site/content/docs/bindings
|
||||
echo "previewdocs.nosqlbench.io" > nosqlbench-build-docs/site/staticCNAME
|
||||
cd nosqlbench-build-docs
|
||||
git add -A
|
||||
CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
|
||||
echo "found $CHANGES to push for doc updates"
|
||||
if (( $CHANGES > 0 ))
|
||||
then
|
||||
git commit -m"docs update for $GITHUB_REF"
|
||||
git push
|
||||
fi
|
||||
echo "push completed"
|
||||
|
@ -1,3 +1,5 @@
|
||||
# Pulsar
|
||||
|
||||
- [1. Overview](#1-overview)
|
||||
- [1.1. Issues Tracker](#11-issues-tracker)
|
||||
- [2. NB Pulsar Driver Workload Definition Yaml File - High Level Structure](#2-nb-pulsar-driver-workload-definition-yaml-file---high-level-structure)
|
@ -1,3 +1,4 @@
|
||||
# Table of contents
|
||||
- [1. Overview](#1-overview)
|
||||
- [1.1. Issues Tracker](#11-issues-tracker)
|
||||
- [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload)
|
||||
|
@ -1,3 +1,4 @@
|
||||
# S4J Adapter
|
||||
- [1. Overview](#1-overview)
|
||||
- [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload)
|
||||
- [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file)
|
||||
|
@ -19,23 +19,27 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
|
||||
import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
|
||||
import io.nosqlbench.api.docsapi.Docs;
|
||||
import io.nosqlbench.api.docsapi.DocsBinder;
|
||||
import io.nosqlbench.api.docsapi.DocsNameSpace;
|
||||
import io.nosqlbench.nb.annotations.Maturity;
|
||||
import io.nosqlbench.nb.annotations.Service;
|
||||
import io.nosqlbench.api.spi.SimpleServiceLoader;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Service(value = BundledMarkdownManifest.class, selector = "adapter-docs")
|
||||
@Service(value = BundledMarkdownManifest.class, selector = "drivers")
|
||||
public class BundledDriverAdapterDocs implements BundledMarkdownManifest {
|
||||
@Override
|
||||
public DocsBinder getDocs() {
|
||||
Docs docs = new Docs().namespace("adapter-docs");
|
||||
DocsBinder docs = new Docs();
|
||||
SimpleServiceLoader<DriverAdapter> loader = new SimpleServiceLoader<>(DriverAdapter.class, Maturity.Any);
|
||||
List<SimpleServiceLoader.Component<? extends DriverAdapter>> namedProviders = loader.getNamedProviders();
|
||||
for (SimpleServiceLoader.Component<? extends DriverAdapter> namedProvider : namedProviders) {
|
||||
DriverAdapter driverAdapter = namedProvider.provider.get();
|
||||
DocsBinder bundledDocs = driverAdapter.getBundledDocs();
|
||||
docs.merge(bundledDocs);
|
||||
docs = docs.merge(bundledDocs);
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
@ -28,11 +28,15 @@ import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.api.content.Content;
|
||||
import io.nosqlbench.api.content.NBIO;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.LongFunction;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* <P>The DriverAdapter interface is expected to be the replacement
|
||||
@ -177,7 +181,7 @@ public interface DriverAdapter<OPTYPE extends Op, SPACETYPE> {
|
||||
* @return A {@link DocsBinder} which describes docs to include for a given adapter.
|
||||
*/
|
||||
default DocsBinder getBundledDocs() {
|
||||
Docs docs = new Docs().namespace("adapter-"+this.getAdapterName());
|
||||
Docs docs = new Docs().namespace("drivers");
|
||||
|
||||
String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName();
|
||||
String cp_docspath = "docs/" + this.getAdapterName();
|
||||
@ -185,6 +189,7 @@ public interface DriverAdapter<OPTYPE extends Op, SPACETYPE> {
|
||||
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
|
||||
|
||||
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first();
|
||||
|
||||
maindoc.map(Content::asPath).ifPresent(docs::addPath);
|
||||
|
||||
return docs.asDocsBinder();
|
||||
|
@ -1,85 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar;
|
||||
|
||||
import com.codahale.metrics.Timer;
|
||||
import io.nosqlbench.driver.pulsar.ops.PulsarOp;
|
||||
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
|
||||
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
public class PulsarAction implements SyncAction {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarAction.class);
|
||||
|
||||
private final int slot;
|
||||
private final PulsarActivity activity;
|
||||
int maxTries = 1;
|
||||
|
||||
public PulsarAction(PulsarActivity activity, int slot) {
|
||||
this.activity = activity;
|
||||
this.slot = slot;
|
||||
this.maxTries = activity.getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public int runCycle(long cycle) {
|
||||
|
||||
// let's fail the action if some async operation failed
|
||||
activity.failOnAsyncOperationFailure();
|
||||
|
||||
long start = System.nanoTime();
|
||||
|
||||
PulsarOp pulsarOp;
|
||||
try (Timer.Context ctx = activity.getBindTimer().time()) {
|
||||
LongFunction<? extends PulsarOp> readyPulsarOp = activity.getSequencer().apply(cycle);
|
||||
pulsarOp = readyPulsarOp.apply(cycle);
|
||||
} catch (Exception bindException) {
|
||||
// if diagnostic mode ...
|
||||
activity.getErrorHandler().handleError(bindException, cycle, 0);
|
||||
throw new RuntimeException(
|
||||
"while binding request in cycle " + cycle + ": " + bindException.getMessage(), bindException
|
||||
);
|
||||
}
|
||||
|
||||
for (int i = 0; i < maxTries; i++) {
|
||||
Timer.Context ctx = activity.getExecuteTimer().time();
|
||||
try {
|
||||
// it is up to the pulsarOp to call Context#close when the activity is executed
|
||||
// this allows us to track time for async operations
|
||||
pulsarOp.run(ctx::close);
|
||||
break;
|
||||
} catch (RuntimeException err) {
|
||||
ErrorDetail errorDetail = activity
|
||||
.getErrorHandler()
|
||||
.handleError(err, cycle, System.nanoTime() - start);
|
||||
if (!errorDetail.isRetryable()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
@ -1,319 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
import com.codahale.metrics.Histogram;
|
||||
import com.codahale.metrics.Timer;
|
||||
import io.nosqlbench.driver.pulsar.ops.PulsarOp;
|
||||
import io.nosqlbench.driver.pulsar.ops.ReadyPulsarOp;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarNBClientConf;
|
||||
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
|
||||
import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler;
|
||||
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
|
||||
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
|
||||
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
|
||||
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
|
||||
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
|
||||
import org.apache.commons.lang3.BooleanUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.admin.PulsarAdmin;
|
||||
import org.apache.pulsar.client.admin.PulsarAdminBuilder;
|
||||
import org.apache.pulsar.client.api.*;
|
||||
import org.apache.pulsar.common.schema.KeyValueEncodingType;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
public class PulsarActivity extends SimpleActivity implements ActivityDefObserver {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarActivity.class);
|
||||
|
||||
private Counter bytesCounter;
|
||||
private Histogram messageSizeHistogram;
|
||||
private Timer bindTimer;
|
||||
private Timer executeTimer;
|
||||
private Timer createTransactionTimer;
|
||||
private Timer commitTransactionTimer;
|
||||
|
||||
// Metrics for NB Pulsar driver milestone: https://github.com/nosqlbench/nosqlbench/milestone/11
|
||||
// - end-to-end latency
|
||||
private Histogram e2eMsgProcLatencyHistogram;
|
||||
|
||||
/**
|
||||
* A histogram that tracks payload round-trip-time, based on a user-defined field in some sender
|
||||
* system which can be interpreted as millisecond epoch time in the system's local time zone.
|
||||
* This is paired with a field name of the same type to be extracted and reported in a meteric
|
||||
* named 'payload-rtt'.
|
||||
*/
|
||||
private Histogram payloadRttHistogram;
|
||||
|
||||
// - message out of sequence error counter
|
||||
private Counter msgErrOutOfSeqCounter;
|
||||
// - message loss counter
|
||||
private Counter msgErrLossCounter;
|
||||
// - message duplicate (when dedup is enabled) error counter
|
||||
private Counter msgErrDuplicateCounter;
|
||||
|
||||
private PulsarSpaceCache pulsarCache;
|
||||
|
||||
private PulsarNBClientConf pulsarNBClientConf;
|
||||
private String pulsarSvcUrl;
|
||||
private String webSvcUrl;
|
||||
private PulsarAdmin pulsarAdmin;
|
||||
private PulsarClient pulsarClient;
|
||||
private Schema<?> pulsarSchema;
|
||||
|
||||
private NBErrorHandler errorHandler;
|
||||
private OpSequence<OpDispenser<? extends PulsarOp>> sequencer;
|
||||
private volatile Throwable asyncOperationFailure;
|
||||
private boolean cycleratePerThread;
|
||||
|
||||
public PulsarActivity(ActivityDef activityDef) {
|
||||
super(activityDef);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdownActivity() {
|
||||
super.shutdownActivity();
|
||||
|
||||
if (pulsarCache == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (PulsarSpace pulsarSpace : pulsarCache.getAssociatedPulsarSpace()) {
|
||||
pulsarSpace.shutdownPulsarSpace();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initActivity() {
|
||||
super.initActivity();
|
||||
pulsarCache = new PulsarSpaceCache(this);
|
||||
|
||||
bytesCounter = ActivityMetrics.counter(activityDef, "bytes");
|
||||
messageSizeHistogram = ActivityMetrics.histogram(activityDef, "message_size", this.getHdrDigits());
|
||||
bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits());
|
||||
executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits());
|
||||
createTransactionTimer = ActivityMetrics.timer(activityDef, "create_transaction", this.getHdrDigits());
|
||||
commitTransactionTimer = ActivityMetrics.timer(activityDef, "commit_transaction", this.getHdrDigits());
|
||||
|
||||
e2eMsgProcLatencyHistogram = ActivityMetrics.histogram(activityDef, "e2e_msg_latency", this.getHdrDigits());
|
||||
payloadRttHistogram = ActivityMetrics.histogram(activityDef, "payload_rtt", this.getHdrDigits());
|
||||
|
||||
msgErrOutOfSeqCounter = ActivityMetrics.counter(activityDef, "err_msg_oos");
|
||||
msgErrLossCounter = ActivityMetrics.counter(activityDef, "err_msg_loss");
|
||||
msgErrDuplicateCounter = ActivityMetrics.counter(activityDef, "err_msg_dup");
|
||||
|
||||
String pulsarClntConfFile =
|
||||
activityDef.getParams().getOptionalString("config").orElse("config.properties");
|
||||
pulsarNBClientConf = new PulsarNBClientConf(pulsarClntConfFile);
|
||||
|
||||
pulsarSvcUrl =
|
||||
activityDef.getParams().getOptionalString("service_url").orElse("pulsar://localhost:6650");
|
||||
webSvcUrl =
|
||||
activityDef.getParams().getOptionalString("web_url").orElse("http://localhost:8080");
|
||||
|
||||
initPulsarAdminAndClientObj();
|
||||
createPulsarSchemaFromConf();
|
||||
|
||||
|
||||
this.sequencer = createOpSequence((ot) -> new ReadyPulsarOp(ot, pulsarCache, this), false, Optional.empty());
|
||||
setDefaultsFromOpSequence(sequencer);
|
||||
onActivityDefUpdate(activityDef);
|
||||
|
||||
this.errorHandler = new NBErrorHandler(
|
||||
() -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
|
||||
this::getExceptionMetrics
|
||||
);
|
||||
|
||||
cycleratePerThread = activityDef.getParams().takeBoolOrDefault("cyclerate_per_thread", false);
|
||||
}
|
||||
|
||||
private final ThreadLocal<RateLimiter> cycleLimiterThreadLocal = ThreadLocal.withInitial(() -> {
|
||||
if (super.getCycleLimiter() != null) {
|
||||
return RateLimiters.createOrUpdate(this.getActivityDef(), "cycles", null,
|
||||
super.getCycleLimiter().getRateSpec());
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
@Override
|
||||
public RateLimiter getCycleLimiter() {
|
||||
if (cycleratePerThread) {
|
||||
return cycleLimiterThreadLocal.get();
|
||||
} else {
|
||||
return super.getCycleLimiter();
|
||||
}
|
||||
}
|
||||
|
||||
public NBErrorHandler getErrorHandler() { return errorHandler; }
|
||||
|
||||
public OpSequence<OpDispenser<? extends PulsarOp>> getSequencer() { return sequencer; }
|
||||
|
||||
public void failOnAsyncOperationFailure() {
|
||||
if (asyncOperationFailure != null) {
|
||||
throw new RuntimeException(asyncOperationFailure);
|
||||
}
|
||||
}
|
||||
|
||||
public void asyncOperationFailed(Throwable ex) {
|
||||
this.asyncOperationFailure = ex;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize
|
||||
* - PulsarAdmin object for adding/deleting tenant, namespace, and topic
|
||||
* - PulsarClient object for message publishing and consuming
|
||||
*/
|
||||
private void initPulsarAdminAndClientObj() {
|
||||
PulsarAdminBuilder adminBuilder =
|
||||
PulsarAdmin.builder()
|
||||
.serviceHttpUrl(webSvcUrl);
|
||||
|
||||
ClientBuilder clientBuilder = PulsarClient.builder();
|
||||
|
||||
try {
|
||||
Map<String, Object> clientConfMap = pulsarNBClientConf.getClientConfMap();
|
||||
|
||||
// Override "client.serviceUrl" setting in config.properties
|
||||
clientConfMap.remove("serviceUrl");
|
||||
clientBuilder.loadConf(clientConfMap).serviceUrl(pulsarSvcUrl);
|
||||
|
||||
// Pulsar Authentication
|
||||
String authPluginClassName =
|
||||
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.authPulginClassName.label);
|
||||
String authParams =
|
||||
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.authParams.label);
|
||||
|
||||
if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) {
|
||||
adminBuilder.authentication(authPluginClassName, authParams);
|
||||
clientBuilder.authentication(authPluginClassName, authParams);
|
||||
}
|
||||
|
||||
String useTlsStr =
|
||||
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.useTls.label);
|
||||
boolean useTls = BooleanUtils.toBoolean(useTlsStr);
|
||||
|
||||
String tlsTrustCertsFilePath =
|
||||
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label);
|
||||
|
||||
String tlsAllowInsecureConnectionStr =
|
||||
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label);
|
||||
boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr);
|
||||
|
||||
String tlsHostnameVerificationEnableStr =
|
||||
(String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label);
|
||||
boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr);
|
||||
|
||||
if ( useTls ) {
|
||||
adminBuilder
|
||||
.enableTlsHostnameVerification(tlsHostnameVerificationEnable);
|
||||
|
||||
clientBuilder
|
||||
.enableTlsHostnameVerification(tlsHostnameVerificationEnable);
|
||||
|
||||
if (!StringUtils.isBlank(tlsTrustCertsFilePath)) {
|
||||
adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath);
|
||||
clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
// Put this outside "if (useTls)" block for easier handling of "tlsAllowInsecureConnection"
|
||||
adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection);
|
||||
clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection);
|
||||
|
||||
pulsarAdmin = adminBuilder.build();
|
||||
pulsarClient = clientBuilder.build();
|
||||
|
||||
////////////////
|
||||
// Not supported in Pulsar 2.8.0
|
||||
//
|
||||
// ClientConfigurationData configurationData = pulsarAdmin.getClientConfigData();
|
||||
// logger.debug(configurationData.toString());
|
||||
|
||||
} catch (PulsarClientException e) {
|
||||
logger.error("Fail to create PulsarAdmin and/or PulsarClient object from the global configuration!");
|
||||
throw new RuntimeException("Fail to create PulsarAdmin and/or PulsarClient object from global configuration!");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Pulsar schema from the definition string
|
||||
*/
|
||||
private void createPulsarSchemaFromConf() {
|
||||
pulsarSchema = buldSchemaFromDefinition("schema.type", "schema.definition");
|
||||
|
||||
// this is to allow KEY_VALUE schema
|
||||
if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) {
|
||||
Schema<?> pulsarKeySchema = buldSchemaFromDefinition("schema.key.type", "schema.key.definition");
|
||||
Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype");
|
||||
KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED;
|
||||
if (encodingType != null) {
|
||||
keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString());
|
||||
}
|
||||
pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType);
|
||||
}
|
||||
}
|
||||
|
||||
private Schema<?> buldSchemaFromDefinition(String schemaTypeConfEntry,
|
||||
String schemaDefinitionConfEntry) {
|
||||
Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry);
|
||||
Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry);
|
||||
String schemaType = (value != null) ? value.toString() : "";
|
||||
|
||||
Schema<?> result;
|
||||
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType)) {
|
||||
String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : "";
|
||||
result = PulsarActivityUtil.getAvroSchema(schemaType, schemaDefStr);
|
||||
} else if (PulsarActivityUtil.isPrimitiveSchemaTypeStr(schemaType)) {
|
||||
result = PulsarActivityUtil.getPrimitiveTypeSchema(schemaType);
|
||||
} else if (PulsarActivityUtil.isAutoConsumeSchemaTypeStr(schemaType)) {
|
||||
result = Schema.AUTO_CONSUME();
|
||||
} else {
|
||||
throw new RuntimeException("Unsupported schema type string: " + schemaType + "; " +
|
||||
"Only primitive type, Avro type and AUTO_CONSUME are supported at the moment!");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public PulsarNBClientConf getPulsarConf() { return this.pulsarNBClientConf;}
|
||||
public String getPulsarSvcUrl() { return this.pulsarSvcUrl;}
|
||||
public String getWebSvcUrl() { return this.webSvcUrl; }
|
||||
public PulsarAdmin getPulsarAdmin() { return this.pulsarAdmin; }
|
||||
public PulsarClient getPulsarClient() { return this.pulsarClient; }
|
||||
public Schema<?> getPulsarSchema() { return pulsarSchema; }
|
||||
|
||||
public Counter getBytesCounter() { return bytesCounter; }
|
||||
public Histogram getMessageSizeHistogram() { return messageSizeHistogram; }
|
||||
public Timer getBindTimer() { return bindTimer; }
|
||||
public Timer getExecuteTimer() { return this.executeTimer; }
|
||||
public Timer getCreateTransactionTimer() { return createTransactionTimer; }
|
||||
public Timer getCommitTransactionTimer() { return commitTransactionTimer; }
|
||||
|
||||
public Histogram getPayloadRttHistogram() {return payloadRttHistogram;}
|
||||
public Histogram getE2eMsgProcLatencyHistogram() { return e2eMsgProcLatencyHistogram; }
|
||||
public Counter getMsgErrOutOfSeqCounter() { return msgErrOutOfSeqCounter; }
|
||||
public Counter getMsgErrLossCounter() { return msgErrLossCounter; }
|
||||
public Counter getMsgErrDuplicateCounter() { return msgErrDuplicateCounter; }
|
||||
}
|
@ -1,52 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar;
|
||||
|
||||
import io.nosqlbench.engine.api.activityapi.core.Action;
|
||||
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
|
||||
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
|
||||
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.nb.annotations.Service;
|
||||
|
||||
@Service(value= ActivityType.class, selector="pulsar")
|
||||
public class PulsarActivityType implements ActivityType<PulsarActivity> {
|
||||
|
||||
@Override
|
||||
public ActionDispenser getActionDispenser(PulsarActivity activity) {
|
||||
if (activity.getParams().getOptionalString("async").isPresent()) {
|
||||
throw new RuntimeException("The async pulsar driver is not implemented yet.");
|
||||
}
|
||||
return new PulsarActionDispenser(activity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarActivity getActivity(ActivityDef activityDef) {
|
||||
return new PulsarActivity(activityDef);
|
||||
}
|
||||
|
||||
private static class PulsarActionDispenser implements ActionDispenser {
|
||||
private final PulsarActivity activity;
|
||||
public PulsarActionDispenser(PulsarActivity activity) {
|
||||
this.activity = activity;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Action getAction(int slot) {
|
||||
return new PulsarAction(activity, slot);
|
||||
}
|
||||
}
|
||||
}
|
@ -1,802 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar;
|
||||
|
||||
import com.codahale.metrics.Gauge;
|
||||
import com.codahale.metrics.Timer;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarNBClientConf;
|
||||
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.admin.Clusters;
|
||||
import org.apache.pulsar.client.admin.PulsarAdmin;
|
||||
import org.apache.pulsar.client.admin.PulsarAdminException;
|
||||
import org.apache.pulsar.client.api.*;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* An instance of a pulsar client, along with all the cached objects which are normally
|
||||
* associated with it during a client session in a typical application.
|
||||
* A PulsarSpace is simply a named and cached set of objects which must be used together.
|
||||
*/
|
||||
public class PulsarSpace {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarSpace.class);
|
||||
|
||||
private final String spaceName;
|
||||
|
||||
private final ConcurrentHashMap<String, Producer<?>> producers = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, Consumer<?>> consumers = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, Reader<?>> readers = new ConcurrentHashMap<>();
|
||||
|
||||
private final PulsarActivity pulsarActivity;
|
||||
private final ActivityDef activityDef;
|
||||
|
||||
private final PulsarNBClientConf pulsarNBClientConf;
|
||||
private final String pulsarSvcUrl;
|
||||
private final String webSvcUrl;
|
||||
private final PulsarAdmin pulsarAdmin;
|
||||
private final PulsarClient pulsarClient;
|
||||
private final Schema<?> pulsarSchema;
|
||||
private final Set<String> pulsarClusterMetadata = new HashSet<>();
|
||||
private final Timer createTransactionTimer;
|
||||
|
||||
public PulsarSpace(String name, PulsarActivity pulsarActivity) {
|
||||
this.spaceName = name;
|
||||
this.pulsarActivity = pulsarActivity;
|
||||
|
||||
this.pulsarNBClientConf = pulsarActivity.getPulsarConf();
|
||||
this.pulsarSvcUrl = pulsarActivity.getPulsarSvcUrl();
|
||||
this.webSvcUrl = pulsarActivity.getWebSvcUrl();
|
||||
this.pulsarAdmin = pulsarActivity.getPulsarAdmin();
|
||||
this.pulsarClient = pulsarActivity.getPulsarClient();
|
||||
this.pulsarSchema = pulsarActivity.getPulsarSchema();
|
||||
this.activityDef = pulsarActivity.getActivityDef();
|
||||
this.createTransactionTimer = pulsarActivity.getCreateTransactionTimer();
|
||||
|
||||
try {
|
||||
Clusters clusters = pulsarAdmin.clusters();
|
||||
List<String> stringList = clusters.getClusters();
|
||||
CollectionUtils.addAll(pulsarClusterMetadata, stringList.listIterator());
|
||||
|
||||
} catch (PulsarAdminException e) {
|
||||
// this is okay if you are connecting with a token that does not have access to the
|
||||
// system configuration
|
||||
logger.info("Could not get list of Pulsar Clusters from global configuration: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public PulsarNBClientConf getPulsarClientConf() { return pulsarNBClientConf; }
|
||||
public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; }
|
||||
public PulsarClient getPulsarClient() { return pulsarClient; }
|
||||
public Schema<?> getPulsarSchema() { return pulsarSchema; }
|
||||
public String getPulsarSvcUrl() { return pulsarSvcUrl;}
|
||||
public String getWebSvcUrl() { return webSvcUrl; }
|
||||
public Set<String> getPulsarClusterMetadata() { return pulsarClusterMetadata; }
|
||||
|
||||
|
||||
// Properly shut down all Pulsar objects (producers, consumers, etc.) that are associated with this space
|
||||
public void shutdownPulsarSpace() {
|
||||
try {
|
||||
for (Producer<?> producer : producers.values()) {
|
||||
if (producer != null) producer.close();
|
||||
}
|
||||
|
||||
for (Consumer<?> consumer : consumers.values()) {
|
||||
if (consumer != null) consumer.close();
|
||||
}
|
||||
|
||||
for (Reader<?> reader : readers.values()) {
|
||||
if (reader != null) reader.close();
|
||||
}
|
||||
|
||||
if (pulsarAdmin != null) pulsarAdmin.close();
|
||||
|
||||
if (pulsarClient != null) pulsarClient.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new RuntimeException("Unexpected error when closing Pulsar objects!");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a proper Pulsar API metrics prefix depending on the API type
|
||||
*
|
||||
* @param apiType - Pulsar API type: producer, consumer, reader, etc.
|
||||
* @param apiObjName - actual name of a producer, a consumer, a reader, etc.
|
||||
* @param topicName - topic name
|
||||
* @return String
|
||||
*/
|
||||
private String getPulsarAPIMetricsPrefix(String apiType, String apiObjName, String topicName) {
|
||||
String apiMetricsPrefix;
|
||||
|
||||
if (!PulsarActivityUtil.isValidPulsarApiType(apiType)) {
|
||||
throw new RuntimeException(
|
||||
"Incorrect Pulsar API type. Valid type list: " + PulsarActivityUtil.getValidPulsarApiTypeList());
|
||||
}
|
||||
|
||||
if (!StringUtils.isBlank(apiObjName)) {
|
||||
apiMetricsPrefix = apiObjName + "_";
|
||||
}
|
||||
else {
|
||||
// we want a meaningful name for the API object (producer, consumer, reader, etc.)
|
||||
// we are not appending the topic name
|
||||
apiMetricsPrefix = apiType;
|
||||
|
||||
if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label))
|
||||
apiMetricsPrefix += producers.size();
|
||||
else if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.CONSUMER.label))
|
||||
apiMetricsPrefix += consumers.size();
|
||||
else if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.READER.label))
|
||||
apiMetricsPrefix += readers.size();
|
||||
|
||||
apiMetricsPrefix += "_";
|
||||
}
|
||||
|
||||
apiMetricsPrefix += topicName + "_";
|
||||
apiMetricsPrefix = apiMetricsPrefix
|
||||
// default name for tests/demos (in all Pulsar examples) is persistent://public/default/test -> use just the topic name test
|
||||
.replace("persistent://public/default/", "")
|
||||
// always remove topic type
|
||||
.replace("non-persistent://", "")
|
||||
.replace("persistent://", "")
|
||||
// persistent://tenant/namespace/topicname -> tenant_namespace_topicname
|
||||
.replace("/","_");
|
||||
|
||||
return apiMetricsPrefix;
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////
|
||||
// Producer Processing --> start
|
||||
//////////////////////////////////////
|
||||
//
|
||||
private static class ProducerGaugeImpl implements Gauge<Object> {
|
||||
private final Producer<?> producer;
|
||||
private final Function<ProducerStats, Object> valueExtractor;
|
||||
|
||||
ProducerGaugeImpl(Producer<?> producer, Function<ProducerStats, Object> valueExtractor) {
|
||||
this.producer = producer;
|
||||
this.valueExtractor = valueExtractor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getValue() {
|
||||
// see Pulsar bug https://github.com/apache/pulsar/issues/10100
|
||||
// we need to synchronize on producer otherwise we could receive corrupted data
|
||||
synchronized(producer) {
|
||||
return valueExtractor.apply(producer.getStats());
|
||||
}
|
||||
}
|
||||
}
|
||||
static Gauge<Object> producerSafeExtractMetric(Producer<?> producer, Function<ProducerStats, Object> valueExtractor) {
|
||||
return new ProducerGaugeImpl(producer, valueExtractor);
|
||||
}
|
||||
|
||||
// Producer name is NOT mandatory
|
||||
// - It can be set at either global level or cycle level
|
||||
// - If set at both levels, cycle level setting takes precedence
|
||||
private String getEffectiveProducerName(String cycleProducerName) {
|
||||
if (!StringUtils.isBlank(cycleProducerName)) {
|
||||
return cycleProducerName;
|
||||
}
|
||||
|
||||
String globalProducerName = pulsarNBClientConf.getProducerName();
|
||||
if (!StringUtils.isBlank(globalProducerName)) {
|
||||
return globalProducerName;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
public Supplier<Transaction> getTransactionSupplier() {
|
||||
PulsarClient pulsarClient = getPulsarClient();
|
||||
return () -> {
|
||||
try (Timer.Context time = createTransactionTimer.time() ){
|
||||
return pulsarClient
|
||||
.newTransaction()
|
||||
.build()
|
||||
.get();
|
||||
} catch (ExecutionException | InterruptedException err) {
|
||||
if (logger.isWarnEnabled()) {
|
||||
logger.warn("Error while starting a new transaction", err);
|
||||
}
|
||||
throw new RuntimeException(err);
|
||||
} catch (PulsarClientException err) {
|
||||
throw new RuntimeException("Transactions are not enabled on Pulsar Client, " +
|
||||
"please set client.enableTransaction=true in your Pulsar Client configuration");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Topic name IS mandatory
|
||||
// - It must be set at either global level or cycle level
|
||||
// - If set at both levels, cycle level setting takes precedence
|
||||
private String getEffectiveProducerTopicName(String cycleTopicName) {
|
||||
if (!StringUtils.isBlank(cycleTopicName)) {
|
||||
return cycleTopicName;
|
||||
}
|
||||
|
||||
String globalTopicName = pulsarNBClientConf.getProducerTopicName();
|
||||
if (!StringUtils.isBlank(globalTopicName)) {
|
||||
return globalTopicName;
|
||||
}
|
||||
|
||||
throw new RuntimeException("Producer topic name must be set at either global level or cycle level!");
|
||||
}
|
||||
|
||||
public Producer<?> getProducer(String cycleTopicName, String cycleProducerName) {
|
||||
String topicName = getEffectiveProducerTopicName(cycleTopicName);
|
||||
String producerName = getEffectiveProducerName(cycleProducerName);
|
||||
|
||||
if (StringUtils.isBlank(topicName)) {
|
||||
throw new RuntimeException("Producer:: must specify a topic name");
|
||||
}
|
||||
|
||||
String producerCacheKey = PulsarActivityUtil.buildCacheKey(producerName, topicName);
|
||||
Producer<?> producer = producers.get(producerCacheKey);
|
||||
|
||||
if (producer == null) {
|
||||
PulsarClient pulsarClient = getPulsarClient();
|
||||
|
||||
// Get other possible producer settings that are set at global level
|
||||
Map<String, Object> producerConf = pulsarNBClientConf.getProducerConfMap();
|
||||
|
||||
// Remove global level settings: "topicName" and "producerName"
|
||||
producerConf.remove(PulsarActivityUtil.PRODUCER_CONF_STD_KEY.topicName.label);
|
||||
producerConf.remove(PulsarActivityUtil.PRODUCER_CONF_STD_KEY.producerName.label);
|
||||
|
||||
String producerMetricsPrefix = getPulsarAPIMetricsPrefix(
|
||||
PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label,
|
||||
producerName,
|
||||
topicName);
|
||||
|
||||
try {
|
||||
ProducerBuilder<?> producerBuilder = pulsarClient.
|
||||
newProducer(pulsarSchema).
|
||||
loadConf(producerConf).
|
||||
topic(topicName);
|
||||
|
||||
if (!StringUtils.isAnyBlank(producerName)) {
|
||||
producerBuilder = producerBuilder.producerName(producerName);
|
||||
}
|
||||
|
||||
producer = producerBuilder.create();
|
||||
producers.put(producerCacheKey, producer);
|
||||
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
producerMetricsPrefix + "total_bytes_sent",
|
||||
producerSafeExtractMetric(producer, (s -> s.getTotalBytesSent() + s.getNumBytesSent())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
producerMetricsPrefix + "total_msg_sent",
|
||||
producerSafeExtractMetric(producer, (s -> s.getTotalMsgsSent() + s.getNumMsgsSent())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
producerMetricsPrefix + "total_send_failed",
|
||||
producerSafeExtractMetric(producer, (s -> s.getTotalSendFailed() + s.getNumSendFailed())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
producerMetricsPrefix + "total_ack_received",
|
||||
producerSafeExtractMetric(producer,(s -> s.getTotalAcksReceived() + s.getNumAcksReceived())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
producerMetricsPrefix + "send_bytes_rate",
|
||||
producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
producerMetricsPrefix + "send_msg_rate",
|
||||
producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate));
|
||||
}
|
||||
catch (PulsarClientException ple) {
|
||||
throw new RuntimeException("Unable to create a Pulsar producer!", ple);
|
||||
}
|
||||
}
|
||||
|
||||
return producer;
|
||||
}
|
||||
//
|
||||
//////////////////////////////////////
|
||||
// Producer Processing <-- end
|
||||
//////////////////////////////////////
|
||||
|
||||
|
||||
//////////////////////////////////////
|
||||
// Consumer Processing --> start
|
||||
//////////////////////////////////////
|
||||
//
|
||||
private static class ConsumerGaugeImpl implements Gauge<Object> {
|
||||
private final Consumer<?> consumer;
|
||||
private final Function<ConsumerStats, Object> valueExtractor;
|
||||
|
||||
ConsumerGaugeImpl(Consumer<?> consumer, Function<ConsumerStats, Object> valueExtractor) {
|
||||
this.consumer = consumer;
|
||||
this.valueExtractor = valueExtractor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getValue() {
|
||||
// see Pulsar bug https://github.com/apache/pulsar/issues/10100
|
||||
// - this is a bug report for producer stats.
|
||||
// - assume this also applies to consumer stats.
|
||||
synchronized(consumer) {
|
||||
return valueExtractor.apply(consumer.getStats());
|
||||
}
|
||||
}
|
||||
}
|
||||
static Gauge<Object> consumerSafeExtractMetric(Consumer<?> consumer, Function<ConsumerStats, Object> valueExtractor) {
|
||||
return new ConsumerGaugeImpl(consumer, valueExtractor);
|
||||
}
|
||||
|
||||
private String getEffectiveSubscriptionName(String cycleSubscriptionName) {
|
||||
if (!StringUtils.isBlank(cycleSubscriptionName)) {
|
||||
return cycleSubscriptionName;
|
||||
}
|
||||
|
||||
String globalSubscriptionName = pulsarNBClientConf.getConsumerSubscriptionName();
|
||||
if (!StringUtils.isBlank(globalSubscriptionName)) {
|
||||
return globalSubscriptionName;
|
||||
}
|
||||
|
||||
throw new RuntimeException("Consumer::Subscription name must be set at either global level or cycle level!");
|
||||
}
|
||||
|
||||
private String getEffectiveSubscriptionTypeStr(String cycleSubscriptionType) {
|
||||
if (!StringUtils.isBlank(cycleSubscriptionType)) {
|
||||
return cycleSubscriptionType;
|
||||
}
|
||||
|
||||
String globalSubscriptionType = pulsarNBClientConf.getConsumerSubscriptionType();
|
||||
if (!StringUtils.isBlank(globalSubscriptionType)) {
|
||||
return globalSubscriptionType;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
private SubscriptionType getEffectiveSubscriptionType(String cycleSubscriptionType) {
|
||||
String effectiveSubscriptionStr = getEffectiveSubscriptionTypeStr(cycleSubscriptionType);
|
||||
SubscriptionType subscriptionType = SubscriptionType.Exclusive;
|
||||
|
||||
if (!StringUtils.isBlank(effectiveSubscriptionStr)) {
|
||||
if (!PulsarActivityUtil.isValidSubscriptionType(effectiveSubscriptionStr)) {
|
||||
throw new RuntimeException("Consumer::Invalid subscription type (\"" +
|
||||
effectiveSubscriptionStr + "\"). \nValid subscription types: " + PulsarActivityUtil.getValidSubscriptionTypeList());
|
||||
} else {
|
||||
subscriptionType = SubscriptionType.valueOf(effectiveSubscriptionStr);
|
||||
}
|
||||
}
|
||||
|
||||
return subscriptionType;
|
||||
}
|
||||
|
||||
private String getEffectiveConsumerName(String cycleConsumerName) {
|
||||
if (!StringUtils.isBlank(cycleConsumerName)) {
|
||||
return cycleConsumerName;
|
||||
}
|
||||
|
||||
String globalConsumerName = pulsarNBClientConf.getConsumerName();
|
||||
if (!StringUtils.isBlank(globalConsumerName)) {
|
||||
return globalConsumerName;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
public Consumer<?> getConsumer(String cycleTopicName,
|
||||
String cycleSubscriptionName,
|
||||
String cycleSubscriptionType,
|
||||
String cycleConsumerName,
|
||||
String cycleKeySharedSubscriptionRanges) {
|
||||
String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName);
|
||||
SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType);
|
||||
String consumerName = getEffectiveConsumerName(cycleConsumerName);
|
||||
|
||||
if (StringUtils.isAnyBlank(cycleTopicName, subscriptionName)) {
|
||||
throw new RuntimeException("Consumer:: must specify a topic name and a subscription name");
|
||||
}
|
||||
|
||||
String consumerCacheKey = PulsarActivityUtil.buildCacheKey(consumerName, subscriptionName, cycleTopicName);
|
||||
Consumer<?> consumer = consumers.get(consumerCacheKey);
|
||||
|
||||
if (consumer == null) {
|
||||
PulsarClient pulsarClient = getPulsarClient();
|
||||
|
||||
// Get other possible consumer settings that are set at global level
|
||||
Map<String, Object> consumerConf = new HashMap<>(pulsarNBClientConf.getConsumerConfMap());
|
||||
|
||||
// Remove global level settings:
|
||||
// - "topicNames", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName"
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
|
||||
// Remove non-standard consumer configuration properties
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
|
||||
|
||||
try {
|
||||
ConsumerBuilder<?> consumerBuilder = pulsarClient.
|
||||
newConsumer(pulsarSchema).
|
||||
loadConf(consumerConf).
|
||||
topic(cycleTopicName).
|
||||
subscriptionName(subscriptionName).
|
||||
subscriptionType(subscriptionType);
|
||||
|
||||
if (subscriptionType == SubscriptionType.Key_Shared) {
|
||||
KeySharedPolicy keySharedPolicy = KeySharedPolicy.autoSplitHashRange();
|
||||
if (cycleKeySharedSubscriptionRanges != null && !cycleKeySharedSubscriptionRanges.isEmpty()) {
|
||||
Range[] ranges = parseRanges(cycleKeySharedSubscriptionRanges);
|
||||
logger.info("Configuring KeySharedPolicy#stickyHashRange with ranges {}", ranges);
|
||||
keySharedPolicy = KeySharedPolicy.stickyHashRange().ranges(ranges);
|
||||
}
|
||||
consumerBuilder.keySharedPolicy(keySharedPolicy);
|
||||
}
|
||||
|
||||
if (!StringUtils.isBlank(consumerName)) {
|
||||
consumerBuilder = consumerBuilder.consumerName(consumerName);
|
||||
}
|
||||
|
||||
consumer = consumerBuilder.subscribe();
|
||||
|
||||
String consumerMetricsPrefix = getPulsarAPIMetricsPrefix(
|
||||
PulsarActivityUtil.PULSAR_API_TYPE.CONSUMER.label,
|
||||
consumerName,
|
||||
cycleTopicName);
|
||||
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "total_bytes_recv",
|
||||
consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "total_msg_recv",
|
||||
consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "total_recv_failed",
|
||||
consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "total_acks_sent",
|
||||
consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "recv_bytes_rate",
|
||||
consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "recv_msg_rate",
|
||||
consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived));
|
||||
} catch (PulsarClientException ple) {
|
||||
ple.printStackTrace();
|
||||
throw new RuntimeException("Unable to create a Pulsar consumer!");
|
||||
}
|
||||
|
||||
consumers.put(consumerCacheKey, consumer);
|
||||
}
|
||||
|
||||
return consumer;
|
||||
}
|
||||
|
||||
private static Range[] parseRanges(String ranges) {
|
||||
if (ranges == null || ranges.isEmpty()) {
|
||||
return new Range[0];
|
||||
}
|
||||
String[] split = ranges.split(",");
|
||||
Range[] result = new Range[split.length];
|
||||
for (int i = 0; i < split.length; i++) {
|
||||
String range = split[i];
|
||||
int pos = range.indexOf("..");
|
||||
if (pos <= 0) {
|
||||
throw new IllegalArgumentException("Invalid range '" + range + "'");
|
||||
}
|
||||
try {
|
||||
int start = Integer.parseInt(range.substring(0, pos));
|
||||
int end = Integer.parseInt(range.substring(pos + 2));
|
||||
result[i] = Range.of(start, end);
|
||||
} catch (NumberFormatException err) {
|
||||
throw new IllegalArgumentException("Invalid range '" + range + "'");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
//////////////////////////////////////
|
||||
// Consumer Processing <-- end
|
||||
//////////////////////////////////////
|
||||
|
||||
|
||||
//////////////////////////////////////
|
||||
// Multi-topic Consumer Processing --> start
|
||||
//////////////////////////////////////
|
||||
//
|
||||
private String getEffectiveConsumerTopicNameListStr(String cycleTopicNames) {
|
||||
if (!StringUtils.isBlank(cycleTopicNames)) {
|
||||
return cycleTopicNames;
|
||||
}
|
||||
|
||||
String globalTopicNames = pulsarNBClientConf.getConsumerTopicNames();
|
||||
if (!StringUtils.isBlank(globalTopicNames)) {
|
||||
return globalTopicNames;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private List<String> getEffectiveConsumerTopicNameList(String cycleTopicNames) {
|
||||
String effectiveTopicNamesStr = getEffectiveConsumerTopicNameListStr(cycleTopicNames);
|
||||
|
||||
String[] names = effectiveTopicNamesStr.split("[;,]");
|
||||
ArrayList<String> effectiveTopicNameList = new ArrayList<>();
|
||||
|
||||
for (String name : names) {
|
||||
if (!StringUtils.isBlank(name))
|
||||
effectiveTopicNameList.add(name.trim());
|
||||
}
|
||||
|
||||
return effectiveTopicNameList;
|
||||
}
|
||||
|
||||
private String getEffectiveConsumerTopicPatternStr(String cycleTopicsPattern) {
|
||||
if (!StringUtils.isBlank(cycleTopicsPattern)) {
|
||||
return cycleTopicsPattern;
|
||||
}
|
||||
|
||||
String globalTopicsPattern = pulsarNBClientConf.getConsumerTopicPattern();
|
||||
if (!StringUtils.isBlank(globalTopicsPattern)) {
|
||||
return globalTopicsPattern;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private Pattern getEffectiveConsumerTopicPattern(String cycleTopicsPattern) {
|
||||
String effectiveTopicsPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicsPattern);
|
||||
Pattern topicsPattern;
|
||||
try {
|
||||
if (!StringUtils.isBlank(effectiveTopicsPatternStr))
|
||||
topicsPattern = Pattern.compile(effectiveTopicsPatternStr);
|
||||
else
|
||||
topicsPattern = null;
|
||||
} catch (PatternSyntaxException pse) {
|
||||
topicsPattern = null;
|
||||
}
|
||||
return topicsPattern;
|
||||
}
|
||||
|
||||
public Consumer<?> getMultiTopicConsumer(
|
||||
String cycleTopicUri,
|
||||
String cycleTopicNameList,
|
||||
String cycleTopicsPattern,
|
||||
String cycleSubscriptionName,
|
||||
String cycleSubscriptionType,
|
||||
String cycleConsumerName) {
|
||||
|
||||
List<String> topicNameList = getEffectiveConsumerTopicNameList(cycleTopicNameList);
|
||||
String topicsPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicsPattern);
|
||||
Pattern topicsPattern = getEffectiveConsumerTopicPattern(cycleTopicsPattern);
|
||||
String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName);
|
||||
SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType);
|
||||
String consumerName = getEffectiveConsumerName(cycleConsumerName);
|
||||
|
||||
if ( subscriptionType.equals(SubscriptionType.Exclusive) && (activityDef.getThreads() > 1) ) {
|
||||
throw new RuntimeException("Consumer:: trying to create multiple consumers of " +
|
||||
"\"Exclusive\" subscription type under the same subscription name to the same topic!");
|
||||
}
|
||||
|
||||
if (StringUtils.isBlank(cycleTopicUri) && topicNameList.isEmpty() && (topicsPattern == null)) {
|
||||
throw new RuntimeException("Consumer:: \"topic_uri\", \"topic_names\" and \"topics_pattern\" parameters can't be all empty/invalid!");
|
||||
}
|
||||
|
||||
// precedence sequence:
|
||||
// topic_names (consumer statement param) >
|
||||
// topics_pattern (consumer statement param) >
|
||||
// topic_uri (document level param)
|
||||
String consumerTopicListString;
|
||||
if (!topicNameList.isEmpty()) {
|
||||
consumerTopicListString = String.join("|", topicNameList);
|
||||
} else if (topicsPattern != null) {
|
||||
consumerTopicListString = topicsPatternStr;
|
||||
} else {
|
||||
consumerTopicListString = cycleTopicUri;
|
||||
}
|
||||
String consumerCacheKey = PulsarActivityUtil.buildCacheKey(
|
||||
consumerName,
|
||||
subscriptionName,
|
||||
consumerTopicListString);
|
||||
|
||||
Consumer<?> consumer = consumers.get(consumerCacheKey);
|
||||
|
||||
if (consumer == null) {
|
||||
PulsarClient pulsarClient = getPulsarClient();
|
||||
|
||||
// Get other possible producer settings that are set at global level
|
||||
Map<String, Object> consumerConf = new HashMap<>(pulsarNBClientConf.getConsumerConfMap());
|
||||
|
||||
// Remove global level settings:
|
||||
// - "topicNameList", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName"
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
|
||||
// Remove non-standard consumer configuration properties
|
||||
consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
|
||||
|
||||
try {
|
||||
ConsumerBuilder<?> consumerBuilder = pulsarClient.newConsumer(pulsarSchema).
|
||||
loadConf(consumerConf).
|
||||
subscriptionName(subscriptionName).
|
||||
subscriptionType(subscriptionType).
|
||||
consumerName(consumerName);
|
||||
|
||||
if (!topicNameList.isEmpty()) {
|
||||
consumerBuilder = consumerBuilder.topics(topicNameList);
|
||||
} else if (topicsPattern != null) {
|
||||
consumerBuilder = consumerBuilder.topicsPattern(topicsPattern);
|
||||
} else {
|
||||
consumerBuilder = consumerBuilder.topic(cycleTopicUri);
|
||||
}
|
||||
|
||||
consumer = consumerBuilder.subscribe();
|
||||
|
||||
String consumerMetricsPrefix = getPulsarAPIMetricsPrefix(
|
||||
PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label,
|
||||
consumerName,
|
||||
consumerTopicListString);
|
||||
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "totalBytesRecvd",
|
||||
consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "totalMsgsRecvd",
|
||||
consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "totalRecvdFailed",
|
||||
consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "totalAcksSent",
|
||||
consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent())));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "recvdBytesRate",
|
||||
consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived));
|
||||
ActivityMetrics.gauge(activityDef,
|
||||
consumerMetricsPrefix + "recvdMsgsRate",
|
||||
consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived));
|
||||
|
||||
} catch (PulsarClientException ple) {
|
||||
ple.printStackTrace();
|
||||
throw new RuntimeException("Unable to create a Pulsar consumer!");
|
||||
}
|
||||
|
||||
consumers.put(consumerCacheKey, consumer);
|
||||
}
|
||||
|
||||
return consumer;
|
||||
}
|
||||
//
|
||||
//////////////////////////////////////
|
||||
// Multi-topic Consumer Processing <-- end
|
||||
//////////////////////////////////////
|
||||
|
||||
|
||||
//////////////////////////////////////
|
||||
// Reader Processing --> Start
|
||||
//////////////////////////////////////
|
||||
private String getEffectiveReaderTopicName(String cycleReaderTopicName) {
|
||||
if (!StringUtils.isBlank(cycleReaderTopicName)) {
|
||||
return cycleReaderTopicName;
|
||||
}
|
||||
|
||||
String globalReaderTopicName = pulsarNBClientConf.getReaderTopicName();
|
||||
if (!StringUtils.isBlank(globalReaderTopicName)) {
|
||||
return globalReaderTopicName;
|
||||
}
|
||||
|
||||
throw new RuntimeException("Reader:: Reader topic name must be set at either global level or cycle level!");
|
||||
}
|
||||
|
||||
private String getEffectiveReaderName(String cycleReaderName) {
|
||||
if (!StringUtils.isBlank(cycleReaderName)) {
|
||||
return cycleReaderName;
|
||||
}
|
||||
|
||||
String globalReaderName = pulsarNBClientConf.getConsumerName();
|
||||
if (!StringUtils.isBlank(globalReaderName)) {
|
||||
return globalReaderName;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private String getEffectiveStartMsgPosStr(String cycleStartMsgPosStr) {
|
||||
if (!StringUtils.isBlank(cycleStartMsgPosStr)) {
|
||||
return cycleStartMsgPosStr;
|
||||
}
|
||||
|
||||
String globalStartMsgPosStr = pulsarNBClientConf.getStartMsgPosStr();
|
||||
if (!StringUtils.isBlank(globalStartMsgPosStr)) {
|
||||
return globalStartMsgPosStr;
|
||||
}
|
||||
|
||||
return PulsarActivityUtil.READER_MSG_POSITION_TYPE.latest.label;
|
||||
}
|
||||
|
||||
public Reader<?> getReader(String cycleTopicName,
|
||||
String cycleReaderName,
|
||||
String cycleStartMsgPos) {
|
||||
|
||||
String topicName = getEffectiveReaderTopicName(cycleTopicName);
|
||||
String readerName = getEffectiveReaderName(cycleReaderName);
|
||||
String startMsgPosStr = getEffectiveStartMsgPosStr(cycleStartMsgPos);
|
||||
if (!PulsarActivityUtil.isValideReaderStartPosition(startMsgPosStr)) {
|
||||
throw new RuntimeException("Reader:: Invalid value for reader start message position!");
|
||||
}
|
||||
|
||||
String readerCacheKey = PulsarActivityUtil.buildCacheKey(topicName, readerName, startMsgPosStr);
|
||||
Reader<?> reader = readers.get(readerCacheKey);
|
||||
|
||||
if (reader == null) {
|
||||
PulsarClient pulsarClient = getPulsarClient();
|
||||
|
||||
Map<String, Object> readerConf = pulsarNBClientConf.getReaderConfMap();
|
||||
|
||||
// Remove global level settings: "topicName" and "readerName"
|
||||
readerConf.remove(PulsarActivityUtil.READER_CONF_STD_KEY.topicName.label);
|
||||
readerConf.remove(PulsarActivityUtil.READER_CONF_STD_KEY.readerName.label);
|
||||
// Remove non-standard reader configuration properties
|
||||
readerConf.remove(PulsarActivityUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label);
|
||||
|
||||
try {
|
||||
ReaderBuilder<?> readerBuilder = pulsarClient.
|
||||
newReader(pulsarSchema).
|
||||
loadConf(readerConf).
|
||||
topic(topicName).
|
||||
readerName(readerName);
|
||||
|
||||
MessageId startMsgId = MessageId.latest;
|
||||
if (startMsgPosStr.equalsIgnoreCase(PulsarActivityUtil.READER_MSG_POSITION_TYPE.earliest.label)) {
|
||||
startMsgId = MessageId.earliest;
|
||||
}
|
||||
//TODO: custom start message position is NOT supported yet
|
||||
//else if (startMsgPosStr.startsWith(PulsarActivityUtil.READER_MSG_POSITION_TYPE.custom.label)) {
|
||||
// startMsgId = MessageId.latest;
|
||||
//}
|
||||
|
||||
reader = readerBuilder.startMessageId(startMsgId).create();
|
||||
|
||||
} catch (PulsarClientException ple) {
|
||||
ple.printStackTrace();
|
||||
throw new RuntimeException("Unable to create a Pulsar reader!");
|
||||
}
|
||||
|
||||
readers.put(readerCacheKey, reader);
|
||||
}
|
||||
|
||||
return reader;
|
||||
}
|
||||
//////////////////////////////////////
|
||||
// Reader Processing <-- end
|
||||
//////////////////////////////////////
|
||||
}
|
@ -1,52 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* To enable flexibility in testing methods, each object graph which is used within
|
||||
* the pulsar API is kept within a single umbrella called the PulsarSpace.
|
||||
* This allows for clients, producers, and consumers to remain connected and
|
||||
* cached in a useful way.
|
||||
*/
|
||||
public class PulsarSpaceCache {
|
||||
|
||||
// TODO: Implement cache limits
|
||||
// TODO: Implement variant cache eviction behaviors (halt, warn, LRU)
|
||||
|
||||
private final PulsarActivity activity;
|
||||
private final ConcurrentHashMap<String, PulsarSpace> clientScopes = new ConcurrentHashMap<>();
|
||||
|
||||
public PulsarSpaceCache(PulsarActivity pulsarActivity) {
|
||||
this.activity = pulsarActivity;
|
||||
}
|
||||
|
||||
public Iterable<PulsarSpace> getAssociatedPulsarSpace() {
|
||||
return clientScopes.values();
|
||||
}
|
||||
|
||||
public PulsarActivity getAssociatedPulsarActivity() {
|
||||
return activity;
|
||||
}
|
||||
|
||||
public PulsarSpace getPulsarSpace(String name) {
|
||||
return clientScopes.computeIfAbsent(name, spaceName -> new PulsarSpace(spaceName, activity));
|
||||
}
|
||||
|
||||
public PulsarActivity getActivity() { return activity; }
|
||||
}
|
@ -1,24 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.exception;
|
||||
|
||||
public class PulsarDriverParamException extends RuntimeException {
|
||||
|
||||
public PulsarDriverParamException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
@ -1,25 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.exception;
|
||||
|
||||
public class PulsarDriverUnexpectedException extends RuntimeException {
|
||||
|
||||
public PulsarDriverUnexpectedException(String message) {
|
||||
super(message);
|
||||
}
|
||||
public PulsarDriverUnexpectedException(Exception e) { super(e); }
|
||||
}
|
@ -1,23 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.exception;
|
||||
|
||||
public class PulsarDriverUnsupportedOpException extends RuntimeException {
|
||||
|
||||
public PulsarDriverUnsupportedOpException() { super("Unsupported Pulsar driver operation type"); }
|
||||
|
||||
}
|
@ -1,24 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
public enum EndToEndStartingTimeSource {
|
||||
NONE, // no end-to-end latency calculation
|
||||
MESSAGE_PUBLISH_TIME, // use message publish timestamp
|
||||
MESSAGE_EVENT_TIME, // use message event timestamp
|
||||
MESSAGE_PROPERTY_E2E_STARTING_TIME // use message property called "e2e_starting_time" as the timestamp
|
||||
}
|
@ -1,103 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import java.util.*;
|
||||
import org.apache.commons.lang3.RandomUtils;
|
||||
|
||||
/**
|
||||
* Handles adding a monotonic sequence number to message properties of sent messages
|
||||
*/
|
||||
class MessageSequenceNumberSendingHandler {
|
||||
static final int SIMULATED_ERROR_PROBABILITY_PERCENTAGE = 10;
|
||||
long number = 1;
|
||||
Queue<Long> outOfOrderNumbers;
|
||||
|
||||
public long getNextSequenceNumber(Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> simulatedErrorTypes) {
|
||||
return getNextSequenceNumber(simulatedErrorTypes, SIMULATED_ERROR_PROBABILITY_PERCENTAGE);
|
||||
}
|
||||
|
||||
long getNextSequenceNumber(Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> simulatedErrorTypes, int errorProbabilityPercentage) {
|
||||
simulateError(simulatedErrorTypes, errorProbabilityPercentage);
|
||||
return nextNumber();
|
||||
}
|
||||
|
||||
private void simulateError(Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> simulatedErrorTypes, int errorProbabilityPercentage) {
|
||||
if (!simulatedErrorTypes.isEmpty() && shouldSimulateError(errorProbabilityPercentage)) {
|
||||
int selectIndex = 0;
|
||||
int numberOfErrorTypes = simulatedErrorTypes.size();
|
||||
if (numberOfErrorTypes > 1) {
|
||||
// pick one of the simulated error type randomly
|
||||
selectIndex = RandomUtils.nextInt(0, numberOfErrorTypes);
|
||||
}
|
||||
PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE errorType = simulatedErrorTypes.stream()
|
||||
.skip(selectIndex)
|
||||
.findFirst()
|
||||
.get();
|
||||
switch (errorType) {
|
||||
case OutOfOrder:
|
||||
// simulate message out of order
|
||||
injectMessagesOutOfOrder();
|
||||
break;
|
||||
case MsgDup:
|
||||
// simulate message duplication
|
||||
injectMessageDuplication();
|
||||
break;
|
||||
case MsgLoss:
|
||||
// simulate message loss
|
||||
injectMessageLoss();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldSimulateError(int errorProbabilityPercentage) {
|
||||
// Simulate error with the specified probability
|
||||
return RandomUtils.nextInt(0, 100) < errorProbabilityPercentage;
|
||||
}
|
||||
|
||||
long nextNumber() {
|
||||
if (outOfOrderNumbers != null) {
|
||||
long nextNumber = outOfOrderNumbers.poll();
|
||||
if (outOfOrderNumbers.isEmpty()) {
|
||||
outOfOrderNumbers = null;
|
||||
}
|
||||
return nextNumber;
|
||||
}
|
||||
return number++;
|
||||
}
|
||||
|
||||
void injectMessagesOutOfOrder() {
|
||||
if (outOfOrderNumbers == null) {
|
||||
outOfOrderNumbers = new ArrayDeque<>(Arrays.asList(number + 2, number, number + 1));
|
||||
number += 3;
|
||||
}
|
||||
}
|
||||
|
||||
void injectMessageDuplication() {
|
||||
if (outOfOrderNumbers == null) {
|
||||
number--;
|
||||
}
|
||||
}
|
||||
|
||||
void injectMessageLoss() {
|
||||
if (outOfOrderNumbers == null) {
|
||||
number++;
|
||||
}
|
||||
}
|
||||
}
|
@ -1,46 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
/**
|
||||
* This maps a set of specifier functions to a pulsar operation. The result contains
|
||||
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
|
||||
* retried if needed.
|
||||
*
|
||||
* This function doesn't act *as* the operation. It merely maps the construction logic into
|
||||
* a simple functional type, given the component functions.
|
||||
*
|
||||
* For additional parameterization, the command template is also provided.
|
||||
*/
|
||||
public abstract class PulsarAdminMapper extends PulsarOpMapper {
|
||||
protected final LongFunction<Boolean> adminDelOpFunc;
|
||||
|
||||
protected PulsarAdminMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc) {
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
|
||||
this.adminDelOpFunc = adminDelOpFunc;
|
||||
}
|
||||
}
|
@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
/**
|
||||
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
|
||||
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
|
||||
* retried if needed.
|
||||
*
|
||||
* This function doesn't act *as* the operation. It merely maps the construction logic into
|
||||
* a simple functional type, given the component functions.
|
||||
*
|
||||
* For additional parameterization, the command template is also provided.
|
||||
*/
|
||||
public class PulsarAdminNamespaceMapper extends PulsarAdminMapper {
|
||||
private final LongFunction<String> namespaceFunc;
|
||||
|
||||
public PulsarAdminNamespaceMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc,
|
||||
LongFunction<String> namespaceFunc)
|
||||
{
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc);
|
||||
this.namespaceFunc = namespaceFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
boolean asyncApi = asyncApiFunc.apply(value);
|
||||
boolean adminDelOp = adminDelOpFunc.apply(value);
|
||||
String namespace = namespaceFunc.apply(value);
|
||||
|
||||
return new PulsarAdminNamespaceOp(
|
||||
clientSpace,
|
||||
asyncApi,
|
||||
adminDelOp,
|
||||
namespace);
|
||||
}
|
||||
}
|
@ -1,101 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.admin.Namespaces;
|
||||
import org.apache.pulsar.client.admin.PulsarAdmin;
|
||||
import org.apache.pulsar.client.admin.PulsarAdminException;
|
||||
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public class PulsarAdminNamespaceOp extends PulsarAdminOp {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarAdminNamespaceOp.class);
|
||||
|
||||
private final String fullNsName;
|
||||
|
||||
public PulsarAdminNamespaceOp(PulsarSpace clientSpace,
|
||||
boolean asyncApi,
|
||||
boolean adminDelOp,
|
||||
String fullNsName)
|
||||
{
|
||||
super(clientSpace, asyncApi, adminDelOp);
|
||||
this.fullNsName = fullNsName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// Do nothing if the namespace name is empty
|
||||
if ( StringUtils.isBlank(fullNsName) ) return;
|
||||
|
||||
PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin();
|
||||
Namespaces namespaces = pulsarAdmin.namespaces();
|
||||
|
||||
// Admin API - create tenants and namespaces
|
||||
if (!adminDelOp) {
|
||||
try {
|
||||
if (!asyncApi) {
|
||||
namespaces.createNamespace(fullNsName);
|
||||
logger.trace("Successfully created namespace \"" + fullNsName + "\" synchronously!");
|
||||
} else {
|
||||
CompletableFuture<Void> future = namespaces.createNamespaceAsync(fullNsName);
|
||||
future.whenComplete((unused, throwable) ->
|
||||
logger.trace("Successfully created namespace \"" + fullNsName + "\" asynchronously!"))
|
||||
.exceptionally(ex -> {
|
||||
logger.error("Failed to create namespace \"" + fullNsName + "\" asynchronously!:" + ex.getMessage());
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (PulsarAdminException.ConflictException ce) {
|
||||
// do nothing if the namespace already exists
|
||||
}
|
||||
catch (PulsarAdminException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Unexpected error when creating pulsar namespace: " + fullNsName);
|
||||
}
|
||||
}
|
||||
// Admin API - delete tenants and namespaces
|
||||
else {
|
||||
try {
|
||||
if (!asyncApi) {
|
||||
namespaces.deleteNamespace(fullNsName, true);
|
||||
logger.trace("Successfully deleted namespace \"" + fullNsName + "\" synchronously!");
|
||||
} else {
|
||||
CompletableFuture<Void> future = namespaces.deleteNamespaceAsync(fullNsName, true);
|
||||
future.whenComplete((unused, throwable) ->
|
||||
logger.trace("Successfully deleted namespace \"" + fullNsName + "\" asynchronously!"))
|
||||
.exceptionally(ex -> {
|
||||
logger.error("Failed to delete namespace \"" + fullNsName + "\" asynchronously!");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (PulsarAdminException.NotFoundException nfe) {
|
||||
// do nothing if the namespace doesn't exist
|
||||
}
|
||||
catch (PulsarAdminException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Unexpected error when deleting pulsar namespace: " + fullNsName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,39 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
public abstract class PulsarAdminOp extends SyncPulsarOp {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarAdminOp.class);
|
||||
|
||||
protected final PulsarSpace clientSpace;
|
||||
protected final boolean asyncApi;
|
||||
protected final boolean adminDelOp;
|
||||
|
||||
protected PulsarAdminOp(PulsarSpace clientSpace,
|
||||
boolean asyncApi,
|
||||
boolean adminDelOp)
|
||||
{
|
||||
this.clientSpace = clientSpace;
|
||||
this.asyncApi = asyncApi;
|
||||
this.adminDelOp = adminDelOp;
|
||||
}
|
||||
}
|
@ -1,72 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
/**
|
||||
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
|
||||
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
|
||||
* retried if needed.
|
||||
*
|
||||
* This function doesn't act *as* the operation. It merely maps the construction logic into
|
||||
* a simple functional type, given the component functions.
|
||||
*
|
||||
* For additional parameterization, the command template is also provided.
|
||||
*/
|
||||
public class PulsarAdminTenantMapper extends PulsarAdminMapper {
|
||||
private final LongFunction<Set<String>> adminRolesFunc;
|
||||
private final LongFunction<Set<String>> allowedClustersFunc;
|
||||
private final LongFunction<String> tenantFunc;
|
||||
|
||||
public PulsarAdminTenantMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc,
|
||||
LongFunction<Set<String>> adminRolesFunc,
|
||||
LongFunction<Set<String>> allowedClustersFunc,
|
||||
LongFunction<String> tenantFunc)
|
||||
{
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc);
|
||||
this.adminRolesFunc = adminRolesFunc;
|
||||
this.allowedClustersFunc = allowedClustersFunc;
|
||||
this.tenantFunc = tenantFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
boolean asyncApi = asyncApiFunc.apply(value);
|
||||
boolean adminDelOp = adminDelOpFunc.apply(value);
|
||||
Set<String> adminRoleSet = adminRolesFunc.apply(value);
|
||||
Set<String> allowedClusterSet = allowedClustersFunc.apply(value);
|
||||
String tenant = tenantFunc.apply(value);
|
||||
|
||||
return new PulsarAdminTenantOp(
|
||||
clientSpace,
|
||||
asyncApi,
|
||||
adminDelOp,
|
||||
adminRoleSet,
|
||||
allowedClusterSet,
|
||||
tenant);
|
||||
}
|
||||
}
|
@ -1,128 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.admin.*;
|
||||
import org.apache.pulsar.common.policies.data.TenantInfo;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public class PulsarAdminTenantOp extends PulsarAdminOp {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarAdminTenantOp.class);
|
||||
|
||||
private final Set<String> adminRoleSet;
|
||||
private final Set<String> allowedClusterSet;
|
||||
private final String tenant;
|
||||
|
||||
public PulsarAdminTenantOp(PulsarSpace clientSpace,
|
||||
boolean asyncApi,
|
||||
boolean adminDelOp,
|
||||
Set<String> adminRoleSet,
|
||||
Set<String> allowedClusterSet,
|
||||
String tenant)
|
||||
{
|
||||
super(clientSpace, asyncApi, adminDelOp);
|
||||
this.adminRoleSet = adminRoleSet;
|
||||
this.allowedClusterSet = allowedClusterSet;
|
||||
this.tenant = tenant;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// Do nothing if the tenant name is empty
|
||||
if ( StringUtils.isBlank(tenant) ) return;
|
||||
|
||||
PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin();
|
||||
Tenants tenants = pulsarAdmin.tenants();
|
||||
Namespaces namespaces = pulsarAdmin.namespaces();
|
||||
|
||||
// Admin API - create tenants and namespaces
|
||||
if (!adminDelOp) {
|
||||
TenantInfo tenantInfo = TenantInfo.builder()
|
||||
.adminRoles(adminRoleSet)
|
||||
.allowedClusters(!allowedClusterSet.isEmpty() ? allowedClusterSet : clientSpace.getPulsarClusterMetadata())
|
||||
.build();
|
||||
|
||||
try {
|
||||
if (!asyncApi) {
|
||||
tenants.createTenant(tenant, tenantInfo);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Successful sync creation of tenant {}", tenant);
|
||||
}
|
||||
} else {
|
||||
CompletableFuture<Void> future = tenants.createTenantAsync(tenant, tenantInfo);
|
||||
future.whenComplete((unused, throwable) -> {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Successful async creation of tenant {}", tenant);
|
||||
}
|
||||
}).exceptionally(ex -> {
|
||||
logger.error("Failed async creation of tenant {}", tenant);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (PulsarAdminException.ConflictException ce) {
|
||||
// do nothing if the tenant already exists
|
||||
}
|
||||
catch (PulsarAdminException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Unexpected error when creating pulsar tenant: " + tenant);
|
||||
}
|
||||
}
|
||||
// Admin API - delete tenants and namespaces
|
||||
else {
|
||||
try {
|
||||
int nsNum = namespaces.getNamespaces(tenant).size();
|
||||
|
||||
// Only delete a tenant when there is no underlying namespaces
|
||||
if ( nsNum == 0 ) {
|
||||
if (!asyncApi) {
|
||||
tenants.deleteTenant(tenant);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Successful sync deletion of tenant {}", tenant);
|
||||
}
|
||||
} else {
|
||||
CompletableFuture<Void> future = tenants.deleteTenantAsync(tenant);
|
||||
future.whenComplete((unused, throwable) -> {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Successful async deletion of tenant {}", tenant);
|
||||
}
|
||||
}).exceptionally(ex -> {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.error("Failed async deletion of tenant {}", tenant);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (PulsarAdminException.NotFoundException nfe) {
|
||||
// do nothing if the tenant doesn't exist
|
||||
}
|
||||
catch (PulsarAdminException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Unexpected error when deleting pulsar tenant: " + tenant);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,91 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.commons.lang3.BooleanUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
/**
|
||||
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
|
||||
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
|
||||
* retried if needed.
|
||||
*
|
||||
* This function doesn't act *as* the operation. It merely maps the construction logic into
|
||||
* a simple functional type, given the component functions.
|
||||
*
|
||||
* For additional parameterization, the command template is also provided.
|
||||
*/
|
||||
public class PulsarAdminTopicMapper extends PulsarAdminMapper {
|
||||
private final LongFunction<String> topicUriFunc;
|
||||
private final LongFunction<String> enablePartionFunc;
|
||||
private final LongFunction<String> partitionNumFunc;
|
||||
|
||||
public PulsarAdminTopicMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc,
|
||||
LongFunction<String> topicUriFunc,
|
||||
LongFunction<String> enablePartionFunc,
|
||||
LongFunction<String> partitionNumFunc)
|
||||
{
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc);
|
||||
this.topicUriFunc = topicUriFunc;
|
||||
this.enablePartionFunc = enablePartionFunc;
|
||||
this.partitionNumFunc = partitionNumFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
String topicUri = topicUriFunc.apply(value);
|
||||
String enablePartitionStr = enablePartionFunc.apply(value);
|
||||
String partitionNumStr = partitionNumFunc.apply(value);
|
||||
boolean asyncApi = asyncApiFunc.apply(value);
|
||||
boolean adminDelOp = adminDelOpFunc.apply(value);
|
||||
|
||||
if ( StringUtils.isBlank(topicUri) ) {
|
||||
throw new RuntimeException("\"topic_uri\" parameter can't be empty when creating a Pulsar topic!");
|
||||
}
|
||||
|
||||
boolean partitionTopic = BooleanUtils.toBoolean(enablePartitionStr);
|
||||
|
||||
boolean invalidPartStr;
|
||||
int partitionNum = 0;
|
||||
if ( StringUtils.isBlank(partitionNumStr) || !StringUtils.isNumeric(partitionNumStr) ) {
|
||||
invalidPartStr = true;
|
||||
} else {
|
||||
partitionNum = Integer.parseInt(partitionNumStr);
|
||||
invalidPartStr = (partitionNum <= 0);
|
||||
}
|
||||
if (partitionTopic && invalidPartStr) {
|
||||
throw new RuntimeException("Invalid specified value for \"partition_num\" parameter when creating partitioned topic!");
|
||||
}
|
||||
|
||||
return new PulsarAdminTopicOp(
|
||||
clientSpace,
|
||||
topicUri,
|
||||
partitionTopic,
|
||||
partitionNum,
|
||||
asyncApi,
|
||||
adminDelOp);
|
||||
}
|
||||
}
|
@ -1,159 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.admin.PulsarAdmin;
|
||||
import org.apache.pulsar.client.admin.PulsarAdminException;
|
||||
import org.apache.pulsar.client.admin.Topics;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
|
||||
public class PulsarAdminTopicOp extends PulsarAdminOp {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarAdminTopicOp.class);
|
||||
|
||||
private final String topicUri;
|
||||
private final boolean partitionTopic;
|
||||
private final int partitionNum;
|
||||
private final String fullNsName;
|
||||
|
||||
public PulsarAdminTopicOp(PulsarSpace clientSpace,
|
||||
String topicUri,
|
||||
boolean partitionTopic,
|
||||
int partitionNum,
|
||||
boolean asyncApi,
|
||||
boolean adminDelOp)
|
||||
{
|
||||
super(clientSpace, asyncApi, adminDelOp);
|
||||
this.topicUri = topicUri;
|
||||
this.partitionTopic = partitionTopic;
|
||||
this.partitionNum = partitionNum;
|
||||
this.fullNsName = PulsarActivityUtil.getFullNamespaceName(this.topicUri);
|
||||
}
|
||||
|
||||
// Check whether the specified topic already exists
|
||||
private boolean checkTopicExistence(Topics topics, String topicUri) {
|
||||
// Check the existence of the topic
|
||||
List<String> topicListWorkingArea = new ArrayList<>();
|
||||
try {
|
||||
if (!partitionTopic) {
|
||||
topicListWorkingArea = topics.getList(fullNsName);
|
||||
}
|
||||
else {
|
||||
topicListWorkingArea = topics.getPartitionedTopicList(fullNsName);
|
||||
}
|
||||
}
|
||||
catch (PulsarAdminException.NotFoundException nfe) {
|
||||
// do nothing
|
||||
}
|
||||
catch (PulsarAdminException e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Failed to retrieve topic info.for pulsar namespace: " + fullNsName);
|
||||
}
|
||||
|
||||
return ( !topicListWorkingArea.isEmpty() && topicListWorkingArea.contains(topicUri) );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin();
|
||||
Topics topics = pulsarAdmin.topics();
|
||||
|
||||
try {
|
||||
// Create the topic
|
||||
if (!adminDelOp) {
|
||||
if (!partitionTopic) {
|
||||
if (!asyncApi) {
|
||||
topics.createNonPartitionedTopic(topicUri);
|
||||
logger.trace("Successfully created non-partitioned topic \"" + topicUri + "\" synchronously!");
|
||||
} else {
|
||||
CompletableFuture<Void> future = topics.createNonPartitionedTopicAsync(topicUri);
|
||||
future.whenComplete((unused, throwable)
|
||||
-> logger.trace("Successfully created non-partitioned topic \"" + topicUri + "\" asynchronously!"))
|
||||
.exceptionally(ex -> {
|
||||
logger.error("Failed to create non-partitioned topic \"" + topicUri + "\" asynchronously!");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if (!asyncApi) {
|
||||
topics.createPartitionedTopic(topicUri, partitionNum);
|
||||
logger.trace("Successfully created partitioned topic \"" + topicUri + "\"" +
|
||||
"(partition_num: " + partitionNum + ") synchronously!");
|
||||
} else {
|
||||
CompletableFuture<Void> future = topics.createPartitionedTopicAsync(topicUri, partitionNum);
|
||||
future.whenComplete((unused, throwable)
|
||||
-> logger.trace("Successfully created partitioned topic \"" + topicUri + "\"" +
|
||||
"(partition_num: " + partitionNum + ") asynchronously!"))
|
||||
.exceptionally(ex -> {
|
||||
logger.error("Failed to create partitioned topic \"" + topicUri + "\"" +
|
||||
"(partition_num: " + partitionNum + ") asynchronously!");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Delete the topic
|
||||
else {
|
||||
if (!partitionTopic) {
|
||||
if (!asyncApi) {
|
||||
topics.delete(topicUri, true);
|
||||
logger.trace("Successfully deleted non-partitioned topic \"" + topicUri + "\" synchronously!");
|
||||
} else {
|
||||
CompletableFuture<Void> future = topics.deleteAsync(topicUri, true);
|
||||
future.whenComplete((unused, throwable)
|
||||
-> logger.trace("Successfully deleted non-partitioned topic \"" + topicUri + "\" asynchronously!"))
|
||||
.exceptionally(ex -> {
|
||||
logger.error("Failed to delete non-partitioned topic \"" + topicUri + "\" asynchronously!");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if (!asyncApi) {
|
||||
topics.deletePartitionedTopic(topicUri, true);
|
||||
logger.trace("Successfully deleted partitioned topic \"" + topicUri + "\" synchronously!");
|
||||
} else {
|
||||
CompletableFuture<Void> future = topics.deletePartitionedTopicAsync(topicUri, true);
|
||||
future.whenComplete((unused, throwable)
|
||||
-> logger.trace("Successfully deleted partitioned topic \"" + topicUri + "\" asynchronously!"))
|
||||
.exceptionally(ex -> {
|
||||
logger.error("Failed to delete partitioned topic \"" + topicUri + "\" asynchronously!");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (PulsarAdminException e) {
|
||||
e.printStackTrace();
|
||||
String errMsg = String.format("Unexpected error when %s pulsar topic: %s (partition topic: %b; partition number: %d)",
|
||||
(!adminDelOp ? "creating" : "deleting"),
|
||||
topicUri,
|
||||
partitionTopic,
|
||||
partitionNum);
|
||||
throw new RuntimeException(errMsg);
|
||||
}
|
||||
}
|
||||
}
|
@ -1,39 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
public class PulsarBatchProducerEndMapper extends PulsarOpMapper {
|
||||
|
||||
public PulsarBatchProducerEndMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc)
|
||||
{
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
return new PulsarBatchProducerEndOp();
|
||||
}
|
||||
}
|
@ -1,50 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.api.errors.BasicError;
|
||||
import org.apache.pulsar.client.api.MessageId;
|
||||
import org.apache.pulsar.client.api.Producer;
|
||||
import org.apache.pulsar.common.util.FutureUtil;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public class PulsarBatchProducerEndOp extends SyncPulsarOp {
|
||||
@Override
|
||||
public void run() {
|
||||
List<CompletableFuture<MessageId>> container = PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.get();
|
||||
Producer<?> producer = PulsarBatchProducerStartOp.threadLocalProducer.get();
|
||||
|
||||
if ((container != null) && (!container.isEmpty())) {
|
||||
try {
|
||||
// producer.flushAsync().get();
|
||||
FutureUtil.waitForAll(container).get();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Batch Producer:: failed to send (some of) the batched messages!");
|
||||
}
|
||||
|
||||
container.clear();
|
||||
PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.set(null);
|
||||
}
|
||||
else {
|
||||
throw new BasicError("You tried to end an empty batch message container. This means you" +
|
||||
" did initiate the batch container properly, or there is an error in your" +
|
||||
" pulsar op sequencing and ratios.");
|
||||
}
|
||||
}
|
||||
}
|
@ -1,77 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
public class PulsarBatchProducerMapper extends PulsarOpMapper {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarBatchProducerMapper.class);
|
||||
|
||||
private final LongFunction<String> keyFunc;
|
||||
private final LongFunction<String> propFunc;
|
||||
private final LongFunction<String> payloadFunc;
|
||||
|
||||
public PulsarBatchProducerMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<String> keyFunc,
|
||||
LongFunction<String> propFunc,
|
||||
LongFunction<String> payloadFunc) {
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
|
||||
this.keyFunc = keyFunc;
|
||||
this.propFunc = propFunc;
|
||||
this.payloadFunc = payloadFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
String msgKey = keyFunc.apply(value);
|
||||
String msgPayload = payloadFunc.apply(value);
|
||||
|
||||
// Check if msgPropJonStr is valid JSON string with a collection of key/value pairs
|
||||
// - if Yes, convert it to a map
|
||||
// - otherwise, log an error message and ignore message properties without throwing a runtime exception
|
||||
Map<String, String> msgProperties = new HashMap<>();
|
||||
String msgPropJsonStr = propFunc.apply(value);
|
||||
try {
|
||||
msgProperties = PulsarActivityUtil.convertJsonToMap(msgPropJsonStr);
|
||||
}
|
||||
catch (Exception e) {
|
||||
logger.error(
|
||||
"PulsarProducerMapper:: Error parsing message property JSON string {}, ignore message properties!",
|
||||
msgPropJsonStr);
|
||||
}
|
||||
|
||||
return new PulsarBatchProducerOp(
|
||||
clientSpace.getPulsarSchema(),
|
||||
msgKey,
|
||||
msgProperties,
|
||||
msgPayload
|
||||
);
|
||||
}
|
||||
}
|
@ -1,83 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.util.AvroUtil;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import org.apache.pulsar.client.api.MessageId;
|
||||
import org.apache.pulsar.client.api.Producer;
|
||||
import org.apache.pulsar.client.api.Schema;
|
||||
import org.apache.pulsar.client.api.TypedMessageBuilder;
|
||||
import org.apache.pulsar.client.api.schema.GenericRecord;
|
||||
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
|
||||
import org.apache.pulsar.common.schema.SchemaType;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public class PulsarBatchProducerOp extends SyncPulsarOp {
|
||||
|
||||
private final Schema<?> pulsarSchema;
|
||||
private final String msgKey;
|
||||
private final Map<String, String> msgProperties;
|
||||
private final String msgPayload;
|
||||
|
||||
public PulsarBatchProducerOp(Schema<?> schema,
|
||||
String key,
|
||||
Map<String, String> msgProperties,
|
||||
String payload) {
|
||||
this.pulsarSchema = schema;
|
||||
this.msgKey = key;
|
||||
this.msgProperties = msgProperties;
|
||||
this.msgPayload = payload;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
if ((msgPayload == null) || msgPayload.isEmpty()) {
|
||||
throw new RuntimeException("Message payload (\"msg-value\") can't be empty!");
|
||||
}
|
||||
|
||||
List<CompletableFuture<MessageId>> container = PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.get();
|
||||
Producer<?> producer = PulsarBatchProducerStartOp.threadLocalProducer.get();
|
||||
assert (producer != null) && (container != null);
|
||||
|
||||
TypedMessageBuilder typedMessageBuilder = producer.newMessage(pulsarSchema);
|
||||
if ((msgKey != null) && (!msgKey.isEmpty())) {
|
||||
typedMessageBuilder = typedMessageBuilder.key(msgKey);
|
||||
}
|
||||
if (!msgProperties.isEmpty()) {
|
||||
typedMessageBuilder = typedMessageBuilder.properties(msgProperties);
|
||||
}
|
||||
|
||||
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
|
||||
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
|
||||
GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro(
|
||||
(GenericAvroSchema) pulsarSchema,
|
||||
pulsarSchema.getSchemaInfo().getSchemaDefinition(),
|
||||
msgPayload
|
||||
);
|
||||
typedMessageBuilder = typedMessageBuilder.value(payload);
|
||||
} else {
|
||||
typedMessageBuilder = typedMessageBuilder.value(msgPayload.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
container.add(typedMessageBuilder.sendAsync());
|
||||
}
|
||||
}
|
@ -1,44 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.pulsar.client.api.Producer;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
public class PulsarBatchProducerStartMapper extends PulsarOpMapper {
|
||||
|
||||
private final LongFunction<Producer<?>> batchProducerFunc;
|
||||
|
||||
public PulsarBatchProducerStartMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Producer<?>> batchProducerFunc) {
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
|
||||
this.batchProducerFunc = batchProducerFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
Producer<?> batchProducer = batchProducerFunc.apply(value);
|
||||
return new PulsarBatchProducerStartOp(batchProducer);
|
||||
}
|
||||
}
|
@ -1,49 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.api.errors.BasicError;
|
||||
import org.apache.commons.compress.utils.Lists;
|
||||
import org.apache.pulsar.client.api.*;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
public class PulsarBatchProducerStartOp extends SyncPulsarOp {
|
||||
|
||||
// TODO: ensure sane container lifecycle management
|
||||
public final transient static ThreadLocal<List<CompletableFuture<MessageId>>> threadLocalBatchMsgContainer = new ThreadLocal<>();
|
||||
public final transient static ThreadLocal<Producer<?>> threadLocalProducer = new ThreadLocal<>();
|
||||
|
||||
public PulsarBatchProducerStartOp(Producer<?> batchProducer) {
|
||||
threadLocalProducer.set(batchProducer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
List<CompletableFuture<MessageId>> container = threadLocalBatchMsgContainer.get();
|
||||
|
||||
if (container == null) {
|
||||
container = Lists.newArrayList();
|
||||
threadLocalBatchMsgContainer.set(container);
|
||||
} else {
|
||||
throw new BasicError("You tried to create a batch message container where one was already" +
|
||||
" defined. This means you did not flush and unset the last container, or there is an error in your" +
|
||||
" pulsar op sequencing and ratios.");
|
||||
}
|
||||
}
|
||||
}
|
@ -1,104 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.api.Consumer;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.function.LongFunction;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
|
||||
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
|
||||
* retried if needed.
|
||||
*
|
||||
* This function doesn't act *as* the operation. It merely maps the construction logic into
|
||||
* a simple functional type, given the component functions.
|
||||
*
|
||||
* For additional parameterization, the command template is also provided.
|
||||
*/
|
||||
public class PulsarConsumerMapper extends PulsarTransactOpMapper {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarProducerMapper.class);
|
||||
|
||||
private final LongFunction<Consumer<?>> consumerFunc;
|
||||
private final EndToEndStartingTimeSource endToEndStartingTimeSource;
|
||||
private final LongFunction<String> payloadRttFieldFunc;
|
||||
|
||||
public PulsarConsumerMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> useTransactionFunc,
|
||||
LongFunction<Boolean> seqTrackingFunc,
|
||||
LongFunction<Supplier<Transaction>> transactionSupplierFunc,
|
||||
LongFunction<Consumer<?>> consumerFunc,
|
||||
EndToEndStartingTimeSource endToEndStartingTimeSource,
|
||||
LongFunction<String> payloadRttFieldFunc) {
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, useTransactionFunc, seqTrackingFunc, transactionSupplierFunc);
|
||||
this.consumerFunc = consumerFunc;
|
||||
this.endToEndStartingTimeSource = endToEndStartingTimeSource;
|
||||
this.payloadRttFieldFunc = payloadRttFieldFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
boolean seqTracking = seqTrackingFunc.apply(value);
|
||||
Consumer<?> consumer = consumerFunc.apply(value);
|
||||
boolean asyncApi = asyncApiFunc.apply(value);
|
||||
boolean useTransaction = useTransactionFunc.apply(value);
|
||||
Supplier<Transaction> transactionSupplier = transactionSupplierFunc.apply(value);
|
||||
String payloadRttFieldFunc = this.payloadRttFieldFunc.apply(value);
|
||||
|
||||
return new PulsarConsumerOp(
|
||||
pulsarActivity,
|
||||
asyncApi,
|
||||
useTransaction,
|
||||
seqTracking,
|
||||
transactionSupplier,
|
||||
consumer,
|
||||
clientSpace.getPulsarSchema(),
|
||||
clientSpace.getPulsarClientConf().getConsumerTimeoutSeconds(),
|
||||
endToEndStartingTimeSource,
|
||||
this::getReceivedMessageSequenceTracker,
|
||||
payloadRttFieldFunc);
|
||||
}
|
||||
|
||||
|
||||
private ReceivedMessageSequenceTracker getReceivedMessageSequenceTracker(String topicName) {
|
||||
return receivedMessageSequenceTrackersForTopicThreadLocal.get()
|
||||
.computeIfAbsent(topicName, k -> createReceivedMessageSequenceTracker());
|
||||
}
|
||||
|
||||
private ReceivedMessageSequenceTracker createReceivedMessageSequenceTracker() {
|
||||
return new ReceivedMessageSequenceTracker(pulsarActivity.getMsgErrOutOfSeqCounter(),
|
||||
pulsarActivity.getMsgErrDuplicateCounter(),
|
||||
pulsarActivity.getMsgErrLossCounter());
|
||||
}
|
||||
|
||||
private final ThreadLocal<Map<String, ReceivedMessageSequenceTracker>> receivedMessageSequenceTrackersForTopicThreadLocal =
|
||||
ThreadLocal.withInitial(HashMap::new);
|
||||
|
||||
}
|
@ -1,303 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
import com.codahale.metrics.Histogram;
|
||||
import com.codahale.metrics.Timer;
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnexpectedException;
|
||||
import io.nosqlbench.driver.pulsar.util.AvroUtil;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.api.*;
|
||||
import org.apache.pulsar.client.api.schema.GenericRecord;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
import org.apache.pulsar.common.schema.SchemaType;
|
||||
|
||||
public class PulsarConsumerOp implements PulsarOp {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarConsumerOp.class);
|
||||
|
||||
private final PulsarActivity pulsarActivity;
|
||||
|
||||
private final boolean asyncPulsarOp;
|
||||
private final boolean useTransaction;
|
||||
private final boolean seqTracking;
|
||||
private final Supplier<Transaction> transactionSupplier;
|
||||
|
||||
private final Consumer<?> consumer;
|
||||
private final Schema<?> pulsarSchema;
|
||||
private final int timeoutSeconds;
|
||||
private final EndToEndStartingTimeSource endToEndStartingTimeSource;
|
||||
|
||||
private final Counter bytesCounter;
|
||||
private final Histogram messageSizeHistogram;
|
||||
private final Timer transactionCommitTimer;
|
||||
|
||||
// keep track of end-to-end message latency
|
||||
private final Histogram e2eMsgProcLatencyHistogram;
|
||||
|
||||
private final Function<String, ReceivedMessageSequenceTracker> receivedMessageSequenceTrackerForTopic;
|
||||
private final Histogram payloadRttHistogram;
|
||||
private final String payloadRttTrackingField;
|
||||
|
||||
private org.apache.avro.Schema avroSchema;
|
||||
|
||||
public PulsarConsumerOp(
|
||||
PulsarActivity pulsarActivity,
|
||||
boolean asyncPulsarOp,
|
||||
boolean useTransaction,
|
||||
boolean seqTracking,
|
||||
Supplier<Transaction> transactionSupplier,
|
||||
Consumer<?> consumer,
|
||||
Schema<?> schema,
|
||||
int timeoutSeconds,
|
||||
EndToEndStartingTimeSource endToEndStartingTimeSource,
|
||||
Function<String, ReceivedMessageSequenceTracker> receivedMessageSequenceTrackerForTopic,
|
||||
String payloadRttTrackingField)
|
||||
{
|
||||
this.pulsarActivity = pulsarActivity;
|
||||
|
||||
this.asyncPulsarOp = asyncPulsarOp;
|
||||
this.useTransaction = useTransaction;
|
||||
this.seqTracking = seqTracking;
|
||||
this.transactionSupplier = transactionSupplier;
|
||||
|
||||
this.consumer = consumer;
|
||||
this.pulsarSchema = schema;
|
||||
this.timeoutSeconds = timeoutSeconds;
|
||||
this.endToEndStartingTimeSource = endToEndStartingTimeSource;
|
||||
|
||||
this.bytesCounter = pulsarActivity.getBytesCounter();
|
||||
this.messageSizeHistogram = pulsarActivity.getMessageSizeHistogram();
|
||||
this.transactionCommitTimer = pulsarActivity.getCommitTransactionTimer();
|
||||
|
||||
this.e2eMsgProcLatencyHistogram = pulsarActivity.getE2eMsgProcLatencyHistogram();
|
||||
this.payloadRttHistogram = pulsarActivity.getPayloadRttHistogram();
|
||||
this.receivedMessageSequenceTrackerForTopic = receivedMessageSequenceTrackerForTopic;
|
||||
this.payloadRttTrackingField = payloadRttTrackingField;
|
||||
}
|
||||
|
||||
private void checkAndUpdateMessageErrorCounter(Message<?> message) {
|
||||
String msgSeqIdStr = message.getProperty(PulsarActivityUtil.MSG_SEQUENCE_NUMBER);
|
||||
|
||||
if ( !StringUtils.isBlank(msgSeqIdStr) ) {
|
||||
long sequenceNumber = Long.parseLong(msgSeqIdStr);
|
||||
ReceivedMessageSequenceTracker receivedMessageSequenceTracker = receivedMessageSequenceTrackerForTopic.apply(message.getTopicName());
|
||||
receivedMessageSequenceTracker.sequenceNumberReceived(sequenceNumber);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(Runnable timeTracker) {
|
||||
|
||||
final Transaction transaction;
|
||||
if (useTransaction) {
|
||||
// if you are in a transaction you cannot set the schema per-message
|
||||
transaction = transactionSupplier.get();
|
||||
}
|
||||
else {
|
||||
transaction = null;
|
||||
}
|
||||
|
||||
if (!asyncPulsarOp) {
|
||||
try {
|
||||
Message<?> message;
|
||||
|
||||
if (timeoutSeconds <= 0) {
|
||||
// wait forever
|
||||
message = consumer.receive();
|
||||
}
|
||||
else {
|
||||
message = consumer
|
||||
.receive(timeoutSeconds, TimeUnit.SECONDS);
|
||||
if (message == null) {
|
||||
throw new TimeoutException("Did not receive a message within "+timeoutSeconds+" seconds");
|
||||
}
|
||||
}
|
||||
|
||||
handleMessage(transaction, message);
|
||||
}
|
||||
catch (Exception e) {
|
||||
logger.error(
|
||||
"Sync message receiving failed - timeout value: {} seconds ", timeoutSeconds, e);
|
||||
throw new PulsarDriverUnexpectedException("" +
|
||||
"Sync message receiving failed - timeout value: " + timeoutSeconds + " seconds ");
|
||||
}
|
||||
}
|
||||
else {
|
||||
try {
|
||||
CompletableFuture<? extends Message<?>> msgRecvFuture = consumer.receiveAsync();
|
||||
if (useTransaction) {
|
||||
// add commit step
|
||||
msgRecvFuture = msgRecvFuture.thenCompose(msg -> {
|
||||
Timer.Context ctx = transactionCommitTimer.time();
|
||||
return transaction
|
||||
.commit()
|
||||
.whenComplete((m,e) -> ctx.close())
|
||||
.thenApply(v-> msg);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
msgRecvFuture.thenAccept(message -> {
|
||||
try {
|
||||
handleMessage(transaction, message);
|
||||
} catch (PulsarClientException | TimeoutException e) {
|
||||
pulsarActivity.asyncOperationFailed(e);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (ExecutionException e) {
|
||||
pulsarActivity.asyncOperationFailed(e.getCause());
|
||||
}
|
||||
}).exceptionally(ex -> {
|
||||
pulsarActivity.asyncOperationFailed(ex);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new PulsarDriverUnexpectedException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void handleMessage(Transaction transaction, Message<?> message)
|
||||
throws PulsarClientException, InterruptedException, ExecutionException, TimeoutException {
|
||||
|
||||
// acknowledge the message as soon as possible
|
||||
if (!useTransaction) {
|
||||
consumer.acknowledgeAsync(message.getMessageId())
|
||||
.get(timeoutSeconds, TimeUnit.SECONDS);
|
||||
} else {
|
||||
consumer.acknowledgeAsync(message.getMessageId(), transaction)
|
||||
.get(timeoutSeconds, TimeUnit.SECONDS);
|
||||
|
||||
// little problem: here we are counting the "commit" time
|
||||
// inside the overall time spent for the execution of the consume operation
|
||||
// we should refactor this operation as for PulsarProducerOp, and use the passed callback
|
||||
// to track with precision the time spent for the operation and for the commit
|
||||
try (Timer.Context ctx = transactionCommitTimer.time()) {
|
||||
transaction.commit().get();
|
||||
}
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
|
||||
|
||||
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
|
||||
org.apache.avro.Schema avroSchema = getSchemaFromConfiguration();
|
||||
org.apache.avro.generic.GenericRecord avroGenericRecord =
|
||||
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData());
|
||||
|
||||
logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}",
|
||||
consumer.getConsumerName(),
|
||||
message.getKey(),
|
||||
message.getProperties(),
|
||||
avroGenericRecord.toString());
|
||||
}
|
||||
else {
|
||||
logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}",
|
||||
consumer.getConsumerName(),
|
||||
message.getKey(),
|
||||
message.getProperties(),
|
||||
new String(message.getData()));
|
||||
}
|
||||
}
|
||||
|
||||
if (!payloadRttTrackingField.isEmpty()) {
|
||||
Object decodedPayload = message.getValue();
|
||||
Long extractedSendTime = null;
|
||||
// if Pulsar is able to decode this it is better to let it do the work
|
||||
// because Pulsar caches the Schema, handles Schema evolution
|
||||
// as much efficiently as possible
|
||||
if (decodedPayload instanceof GenericRecord) {
|
||||
GenericRecord pulsarGenericRecord = (GenericRecord) decodedPayload;
|
||||
Object field = pulsarGenericRecord.getField(payloadRttTrackingField);
|
||||
if (field != null) {
|
||||
if (field instanceof Number) {
|
||||
extractedSendTime = ((Number) field).longValue();
|
||||
} else {
|
||||
extractedSendTime = Long.valueOf(field.toString());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
org.apache.avro.Schema avroSchema = getSchemaFromConfiguration();
|
||||
org.apache.avro.generic.GenericRecord avroGenericRecord =
|
||||
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData());
|
||||
if (avroGenericRecord.hasField(payloadRttTrackingField)) {
|
||||
extractedSendTime = (Long) avroGenericRecord.get(payloadRttTrackingField);
|
||||
}
|
||||
}
|
||||
if (extractedSendTime != null) {
|
||||
long delta = System.currentTimeMillis() - extractedSendTime;
|
||||
payloadRttHistogram.update(delta);
|
||||
}
|
||||
}
|
||||
|
||||
// keep track end-to-end message processing latency
|
||||
if (endToEndStartingTimeSource != EndToEndStartingTimeSource.NONE) {
|
||||
long startTimeStamp = 0L;
|
||||
switch (endToEndStartingTimeSource) {
|
||||
case MESSAGE_PUBLISH_TIME:
|
||||
startTimeStamp = message.getPublishTime();
|
||||
break;
|
||||
case MESSAGE_EVENT_TIME:
|
||||
startTimeStamp = message.getEventTime();
|
||||
break;
|
||||
case MESSAGE_PROPERTY_E2E_STARTING_TIME:
|
||||
String startingTimeProperty = message.getProperty("e2e_starting_time");
|
||||
startTimeStamp = startingTimeProperty != null ? Long.parseLong(startingTimeProperty) : 0L;
|
||||
break;
|
||||
}
|
||||
if (startTimeStamp != 0L) {
|
||||
long e2eMsgLatency = System.currentTimeMillis() - startTimeStamp;
|
||||
e2eMsgProcLatencyHistogram.update(e2eMsgLatency);
|
||||
}
|
||||
}
|
||||
|
||||
// keep track of message errors and update error counters
|
||||
if (seqTracking) checkAndUpdateMessageErrorCounter(message);
|
||||
|
||||
int messageSize = message.getData().length;
|
||||
bytesCounter.inc(messageSize);
|
||||
messageSizeHistogram.update(messageSize);
|
||||
}
|
||||
|
||||
private org.apache.avro.Schema getSchemaFromConfiguration() {
|
||||
String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition();
|
||||
// no need for synchronization, this is only a cache
|
||||
// in case of the race we will parse the string twice, not a big
|
||||
if (avroSchema == null) {
|
||||
avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
|
||||
}
|
||||
return avroSchema;
|
||||
}
|
||||
|
||||
}
|
@ -1,30 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
/**
|
||||
* Base type of all Pulsar Operations including Producers and Consumers.
|
||||
*/
|
||||
public interface PulsarOp {
|
||||
|
||||
/**
|
||||
* Execute the operation, invoke the timeTracker when the operation ended.
|
||||
* The timeTracker can be invoked in a separate thread, it is only used for metrics.
|
||||
* @param timeTracker
|
||||
*/
|
||||
void run(Runnable timeTracker);
|
||||
}
|
@ -1,45 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.pulsar.client.api.Producer;
|
||||
import org.apache.pulsar.client.api.Schema;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public abstract class PulsarOpMapper implements LongFunction<PulsarOp> {
|
||||
protected final CommandTemplate cmdTpl;
|
||||
protected final PulsarSpace clientSpace;
|
||||
protected final PulsarActivity pulsarActivity;
|
||||
protected final LongFunction<Boolean> asyncApiFunc;
|
||||
|
||||
public PulsarOpMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc)
|
||||
{
|
||||
this.cmdTpl = cmdTpl;
|
||||
this.clientSpace = clientSpace;
|
||||
this.pulsarActivity = pulsarActivity;
|
||||
this.asyncApiFunc = asyncApiFunc;
|
||||
}
|
||||
}
|
@ -1,129 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.LongFunction;
|
||||
import java.util.function.Supplier;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.api.Producer;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
|
||||
/**
|
||||
* This maps a set of specifier functions to a pulsar operation. The pulsar operation contains
|
||||
* enough state to define a pulsar operation such that it can be executed, measured, and possibly
|
||||
* retried if needed.
|
||||
*
|
||||
* This function doesn't act *as* the operation. It merely maps the construction logic into
|
||||
* a simple functional type, given the component functions.
|
||||
*
|
||||
* For additional parameterization, the command template is also provided.
|
||||
*/
|
||||
public class PulsarProducerMapper extends PulsarTransactOpMapper {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarProducerMapper.class);
|
||||
|
||||
private final LongFunction<Producer<?>> producerFunc;
|
||||
private final Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> seqErrSimuTypes;
|
||||
private final LongFunction<String> keyFunc;
|
||||
private final LongFunction<String> propFunc;
|
||||
private final LongFunction<String> payloadFunc;
|
||||
|
||||
public PulsarProducerMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> useTransactionFunc,
|
||||
LongFunction<Boolean> seqTrackingFunc,
|
||||
LongFunction<Supplier<Transaction>> transactionSupplierFunc,
|
||||
LongFunction<Producer<?>> producerFunc,
|
||||
Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> seqErrSimuTypes,
|
||||
LongFunction<String> keyFunc,
|
||||
LongFunction<String> propFunc,
|
||||
LongFunction<String> payloadFunc) {
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, useTransactionFunc, seqTrackingFunc, transactionSupplierFunc);
|
||||
|
||||
this.producerFunc = producerFunc;
|
||||
this.seqErrSimuTypes = seqErrSimuTypes;
|
||||
this.keyFunc = keyFunc;
|
||||
this.propFunc = propFunc;
|
||||
this.payloadFunc = payloadFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
boolean asyncApi = asyncApiFunc.apply(value);
|
||||
boolean useTransaction = useTransactionFunc.apply(value);
|
||||
Supplier<Transaction> transactionSupplier = transactionSupplierFunc.apply(value);
|
||||
|
||||
Producer<?> producer = producerFunc.apply(value);
|
||||
|
||||
String msgKey = keyFunc.apply(value);
|
||||
String msgPayload = payloadFunc.apply(value);
|
||||
|
||||
// Check if msgPropJonStr is valid JSON string with a collection of key/value pairs
|
||||
// - if Yes, convert it to a map
|
||||
// - otherwise, log an error message and ignore message properties without throwing a runtime exception
|
||||
Map<String, String> msgProperties = new HashMap<>();
|
||||
String msgPropJsonStr = propFunc.apply(value);
|
||||
if (!StringUtils.isBlank(msgPropJsonStr)) {
|
||||
try {
|
||||
msgProperties = PulsarActivityUtil.convertJsonToMap(msgPropJsonStr);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error(
|
||||
"Error parsing message property JSON string {}, ignore message properties!",
|
||||
msgPropJsonStr);
|
||||
}
|
||||
}
|
||||
|
||||
boolean sequenceTrackingEnabled = seqTrackingFunc.apply(value);
|
||||
if (sequenceTrackingEnabled) {
|
||||
long nextSequenceNumber = getMessageSequenceNumberSendingHandler(producer.getTopic())
|
||||
.getNextSequenceNumber(seqErrSimuTypes);
|
||||
msgProperties.put(PulsarActivityUtil.MSG_SEQUENCE_NUMBER, String.valueOf(nextSequenceNumber));
|
||||
}
|
||||
|
||||
return new PulsarProducerOp(
|
||||
pulsarActivity,
|
||||
asyncApi,
|
||||
useTransaction,
|
||||
transactionSupplier,
|
||||
producer,
|
||||
clientSpace.getPulsarSchema(),
|
||||
msgKey,
|
||||
msgProperties,
|
||||
msgPayload);
|
||||
}
|
||||
|
||||
private MessageSequenceNumberSendingHandler getMessageSequenceNumberSendingHandler(String topicName) {
|
||||
return MessageSequenceNumberSendingHandlersThreadLocal.get()
|
||||
.computeIfAbsent(topicName, k -> new MessageSequenceNumberSendingHandler());
|
||||
}
|
||||
|
||||
private final ThreadLocal<Map<String, MessageSequenceNumberSendingHandler>> MessageSequenceNumberSendingHandlersThreadLocal =
|
||||
ThreadLocal.withInitial(HashMap::new);
|
||||
|
||||
}
|
@ -1,304 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
import com.codahale.metrics.Histogram;
|
||||
import com.codahale.metrics.Timer;
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException;
|
||||
import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnexpectedException;
|
||||
import io.nosqlbench.driver.pulsar.util.AvroUtil;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.api.*;
|
||||
import org.apache.pulsar.client.api.schema.GenericRecord;
|
||||
import org.apache.pulsar.client.api.schema.KeyValueSchema;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
|
||||
import org.apache.pulsar.common.schema.KeyValue;
|
||||
import org.apache.pulsar.common.schema.SchemaType;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public class PulsarProducerOp implements PulsarOp {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarProducerOp.class);
|
||||
|
||||
private final PulsarActivity pulsarActivity;
|
||||
|
||||
private final boolean asyncPulsarOp;
|
||||
private final boolean useTransaction;
|
||||
private final Supplier<Transaction> transactionSupplier;
|
||||
|
||||
private final Producer<?> producer;
|
||||
private final Schema<?> pulsarSchema;
|
||||
private final String msgKey;
|
||||
private final Map<String, String> msgProperties;
|
||||
private final String msgPayload;
|
||||
|
||||
private final Counter bytesCounter;
|
||||
private final Histogram messageSizeHistogram;
|
||||
private final Timer transactionCommitTimer;
|
||||
|
||||
private org.apache.avro.Schema avroSchema;
|
||||
private org.apache.avro.Schema avroKeySchema;
|
||||
|
||||
public PulsarProducerOp( PulsarActivity pulsarActivity,
|
||||
boolean asyncPulsarOp,
|
||||
boolean useTransaction,
|
||||
Supplier<Transaction> transactionSupplier,
|
||||
Producer<?> producer,
|
||||
Schema<?> schema,
|
||||
String key,
|
||||
Map<String, String> msgProperties,
|
||||
String payload) {
|
||||
this.pulsarActivity = pulsarActivity;
|
||||
|
||||
this.asyncPulsarOp = asyncPulsarOp;
|
||||
this.useTransaction = useTransaction;
|
||||
this.transactionSupplier = transactionSupplier;
|
||||
|
||||
this.producer = producer;
|
||||
this.pulsarSchema = schema;
|
||||
this.msgKey = key;
|
||||
this.msgProperties = msgProperties;
|
||||
this.msgPayload = payload;
|
||||
|
||||
this.bytesCounter = pulsarActivity.getBytesCounter();
|
||||
this.messageSizeHistogram = pulsarActivity.getMessageSizeHistogram();
|
||||
this.transactionCommitTimer = pulsarActivity.getCommitTransactionTimer();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(Runnable timeTracker) {
|
||||
if ( StringUtils.isBlank(msgPayload)) {
|
||||
throw new PulsarDriverParamException("Message payload (\"msg-value\") can't be empty!");
|
||||
}
|
||||
|
||||
TypedMessageBuilder typedMessageBuilder;
|
||||
|
||||
final Transaction transaction;
|
||||
if (useTransaction) {
|
||||
// if you are in a transaction you cannot set the schema per-message
|
||||
transaction = transactionSupplier.get();
|
||||
typedMessageBuilder = producer.newMessage(transaction);
|
||||
}
|
||||
else {
|
||||
transaction = null;
|
||||
typedMessageBuilder = producer.newMessage(pulsarSchema);
|
||||
}
|
||||
|
||||
// set message key
|
||||
if (!StringUtils.isBlank(msgKey)) {
|
||||
typedMessageBuilder = typedMessageBuilder.key(msgKey);
|
||||
}
|
||||
|
||||
// set message properties
|
||||
if ( !msgProperties.isEmpty() ) {
|
||||
typedMessageBuilder = typedMessageBuilder.properties(msgProperties);
|
||||
}
|
||||
|
||||
// set message payload
|
||||
int messageSize;
|
||||
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
|
||||
if (pulsarSchema instanceof KeyValueSchema) {
|
||||
|
||||
// {KEY IN JSON}||{VALUE IN JSON}
|
||||
int separator = msgPayload.indexOf("}||{");
|
||||
if (separator < 0) {
|
||||
throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)");
|
||||
}
|
||||
String keyInput = msgPayload.substring(0, separator + 1);
|
||||
String valueInput = msgPayload.substring(separator + 3);
|
||||
|
||||
KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema;
|
||||
org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
|
||||
GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro(
|
||||
(GenericAvroSchema) keyValueSchema.getValueSchema(),
|
||||
avroSchema,
|
||||
valueInput
|
||||
);
|
||||
|
||||
org.apache.avro.Schema avroSchemaForKey = getKeyAvroSchemaFromConfiguration();
|
||||
GenericRecord key = AvroUtil.GetGenericRecord_PulsarAvro(
|
||||
(GenericAvroSchema) keyValueSchema.getKeySchema(),
|
||||
avroSchemaForKey,
|
||||
keyInput
|
||||
);
|
||||
typedMessageBuilder = typedMessageBuilder.value(new KeyValue(key, payload));
|
||||
// TODO: add a way to calculate the message size for KEY_VALUE messages
|
||||
messageSize = msgPayload.length();
|
||||
} else if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
|
||||
GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro(
|
||||
(GenericAvroSchema) pulsarSchema,
|
||||
pulsarSchema.getSchemaInfo().getSchemaDefinition(),
|
||||
msgPayload
|
||||
);
|
||||
typedMessageBuilder = typedMessageBuilder.value(payload);
|
||||
// TODO: add a way to calculate the message size for AVRO messages
|
||||
messageSize = msgPayload.length();
|
||||
} else {
|
||||
byte[] array = msgPayload.getBytes(StandardCharsets.UTF_8);
|
||||
typedMessageBuilder = typedMessageBuilder.value(array);
|
||||
messageSize = array.length;
|
||||
}
|
||||
messageSizeHistogram.update(messageSize);
|
||||
bytesCounter.inc(messageSize);
|
||||
|
||||
//TODO: add error handling with failed message production
|
||||
if (!asyncPulsarOp) {
|
||||
try {
|
||||
logger.trace("Sending message");
|
||||
typedMessageBuilder.send();
|
||||
|
||||
if (useTransaction) {
|
||||
try (Timer.Context ctx = transactionCommitTimer.time()) {
|
||||
transaction.commit().get();
|
||||
}
|
||||
}
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
|
||||
org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
|
||||
org.apache.avro.generic.GenericRecord avroGenericRecord =
|
||||
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload);
|
||||
|
||||
logger.debug("({}) Sync message sent: msg-key={}; msg-properties={}; msg-payload={})",
|
||||
producer.getProducerName(),
|
||||
msgKey,
|
||||
msgProperties,
|
||||
avroGenericRecord.toString());
|
||||
}
|
||||
else {
|
||||
logger.debug("({}) Sync message sent; msg-key={}; msg-properties={}; msg-payload={}",
|
||||
producer.getProducerName(),
|
||||
msgKey,
|
||||
msgProperties,
|
||||
msgPayload);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (PulsarClientException | ExecutionException | InterruptedException pce) {
|
||||
String errMsg =
|
||||
"Sync message sending failed: " +
|
||||
"key - " + msgKey + "; " +
|
||||
"properties - " + msgProperties + "; " +
|
||||
"payload - " + msgPayload;
|
||||
|
||||
logger.trace(errMsg);
|
||||
|
||||
throw new PulsarDriverUnexpectedException(errMsg);
|
||||
}
|
||||
|
||||
timeTracker.run();
|
||||
}
|
||||
else {
|
||||
try {
|
||||
// we rely on blockIfQueueIsFull in order to throttle the request in this case
|
||||
CompletableFuture<?> future = typedMessageBuilder.sendAsync();
|
||||
|
||||
if (useTransaction) {
|
||||
// add commit step
|
||||
future = future.thenCompose(msg -> {
|
||||
Timer.Context ctx = transactionCommitTimer.time();
|
||||
return transaction
|
||||
.commit()
|
||||
.whenComplete((m,e) -> ctx.close())
|
||||
.thenApply(v-> msg);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
future.whenComplete((messageId, error) -> {
|
||||
if (logger.isDebugEnabled()) {
|
||||
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
|
||||
org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration();
|
||||
org.apache.avro.generic.GenericRecord avroGenericRecord =
|
||||
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload);
|
||||
|
||||
logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={})",
|
||||
producer.getProducerName(),
|
||||
msgKey,
|
||||
msgProperties,
|
||||
avroGenericRecord.toString());
|
||||
}
|
||||
else {
|
||||
logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={}",
|
||||
producer.getProducerName(),
|
||||
msgKey,
|
||||
msgProperties,
|
||||
msgPayload);
|
||||
}
|
||||
}
|
||||
|
||||
timeTracker.run();
|
||||
}).exceptionally(ex -> {
|
||||
logger.error("Async message sending failed: " +
|
||||
"key - " + msgKey + "; " +
|
||||
"properties - " + msgProperties + "; " +
|
||||
"payload - " + msgPayload);
|
||||
|
||||
pulsarActivity.asyncOperationFailed(ex);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new PulsarDriverUnexpectedException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private org.apache.avro.Schema getAvroSchemaFromConfiguration() {
|
||||
// no need for synchronization, this is only a cache
|
||||
// in case of the race we will parse the string twice, not a big
|
||||
if (avroSchema == null) {
|
||||
if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
|
||||
KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
|
||||
Schema valueSchema = kvSchema.getValueSchema();
|
||||
String avroDefStr = valueSchema.getSchemaInfo().getSchemaDefinition();
|
||||
avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
|
||||
} else {
|
||||
String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition();
|
||||
avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
|
||||
}
|
||||
}
|
||||
return avroSchema;
|
||||
}
|
||||
|
||||
private org.apache.avro.Schema getKeyAvroSchemaFromConfiguration() {
|
||||
// no need for synchronization, this is only a cache
|
||||
// in case of the race we will parse the string twice, not a big
|
||||
if (avroKeySchema == null) {
|
||||
if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) {
|
||||
KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema;
|
||||
Schema keySchema = kvSchema.getKeySchema();
|
||||
String avroDefStr = keySchema.getSchemaInfo().getSchemaDefinition();
|
||||
avroKeySchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr);
|
||||
} else {
|
||||
throw new RuntimeException("We are not using KEY_VALUE schema, so no Schema for the Key!");
|
||||
}
|
||||
}
|
||||
return avroKeySchema;
|
||||
}
|
||||
}
|
@ -1,51 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.pulsar.client.api.Reader;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
public class PulsarReaderMapper extends PulsarOpMapper {
|
||||
|
||||
private final LongFunction<Reader<?>> readerFunc;
|
||||
|
||||
public PulsarReaderMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Reader<?>> readerFunc)
|
||||
{
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
|
||||
this.readerFunc = readerFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
Reader<?> reader = readerFunc.apply(value);
|
||||
boolean asyncApi = asyncApiFunc.apply(value);
|
||||
|
||||
return new PulsarReaderOp(
|
||||
reader,
|
||||
clientSpace.getPulsarSchema(),
|
||||
asyncApi
|
||||
);
|
||||
}
|
||||
}
|
@ -1,76 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.util.AvroUtil;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import org.apache.pulsar.client.api.Message;
|
||||
import org.apache.pulsar.client.api.PulsarClientException;
|
||||
import org.apache.pulsar.client.api.Reader;
|
||||
import org.apache.pulsar.client.api.Schema;
|
||||
import org.apache.pulsar.common.schema.SchemaType;
|
||||
|
||||
public class PulsarReaderOp extends SyncPulsarOp {
|
||||
private final Reader<?> reader;
|
||||
private final Schema<?> pulsarSchema;
|
||||
private final boolean asyncPulsarOp;
|
||||
|
||||
public PulsarReaderOp(Reader<?> reader, Schema<?> schema, boolean asyncPulsarOp) {
|
||||
this.reader = reader;
|
||||
this.pulsarSchema = schema;
|
||||
this.asyncPulsarOp = asyncPulsarOp;
|
||||
}
|
||||
|
||||
public void syncRead() {
|
||||
try {
|
||||
SchemaType schemaType = pulsarSchema.getSchemaInfo().getType();
|
||||
String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition();
|
||||
|
||||
// TODO: how many messages to read per NB cycle?
|
||||
Message<?> message;
|
||||
while (reader.hasMessageAvailable()) {
|
||||
message = reader.readNext();
|
||||
|
||||
if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) {
|
||||
org.apache.avro.Schema avroSchema =
|
||||
AvroUtil.GetSchema_ApacheAvro(avroDefStr);
|
||||
|
||||
org.apache.avro.generic.GenericRecord avroGenericRecord =
|
||||
AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData());
|
||||
|
||||
System.out.println("msg-key=" + message.getKey() + " msg-payload=" + avroGenericRecord.toString());
|
||||
} else {
|
||||
System.out.println("msg-key=" + message.getKey() + " msg-payload=" + new String(message.getData()));
|
||||
}
|
||||
}
|
||||
} catch (PulsarClientException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void asyncRead() {
|
||||
//TODO: add support for async read
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
if (!asyncPulsarOp)
|
||||
syncRead();
|
||||
else
|
||||
asyncRead();
|
||||
}
|
||||
}
|
@ -1,45 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
|
||||
import java.util.function.LongFunction;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public abstract class PulsarTransactOpMapper extends PulsarOpMapper {
|
||||
protected final LongFunction<Boolean> useTransactionFunc;
|
||||
protected final LongFunction<Boolean> seqTrackingFunc;
|
||||
protected final LongFunction<Supplier<Transaction>> transactionSupplierFunc;
|
||||
|
||||
public PulsarTransactOpMapper(CommandTemplate cmdTpl,
|
||||
PulsarSpace clientSpace,
|
||||
PulsarActivity pulsarActivity,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> useTransactionFunc,
|
||||
LongFunction<Boolean> seqTrackingFunc,
|
||||
LongFunction<Supplier<Transaction>> transactionSupplierFunc)
|
||||
{
|
||||
super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc);
|
||||
this.useTransactionFunc = useTransactionFunc;
|
||||
this.seqTrackingFunc = seqTrackingFunc;
|
||||
this.transactionSupplierFunc = transactionSupplierFunc;
|
||||
}
|
||||
}
|
@ -1,541 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.PulsarActivity;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpace;
|
||||
import io.nosqlbench.driver.pulsar.PulsarSpaceCache;
|
||||
import io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException;
|
||||
import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnsupportedOpException;
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
|
||||
import io.nosqlbench.engine.api.templating.CommandTemplate;
|
||||
import org.apache.commons.lang3.BooleanUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.api.Consumer;
|
||||
import org.apache.pulsar.client.api.Producer;
|
||||
import org.apache.pulsar.client.api.Reader;
|
||||
import org.apache.pulsar.client.api.transaction.Transaction;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.LongFunction;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ReadyPulsarOp extends BaseOpDispenser<PulsarOp> {
|
||||
|
||||
// TODO: Add this to the pulsar driver docs
|
||||
public static final String RTT_TRACKING_FIELD = "payload-tracking-field";
|
||||
private final static Logger logger = LogManager.getLogger(ReadyPulsarOp.class);
|
||||
|
||||
private final OpTemplate opTpl;
|
||||
private final CommandTemplate cmdTpl;
|
||||
private final PulsarSpace clientSpace;
|
||||
private final LongFunction<PulsarOp> opFunc;
|
||||
private final PulsarActivity pulsarActivity;
|
||||
|
||||
// TODO: Add docs for the command template with respect to the OpTemplate
|
||||
|
||||
public ReadyPulsarOp(OpTemplate optpl, PulsarSpaceCache pcache, PulsarActivity pulsarActivity) {
|
||||
super(optpl);
|
||||
// TODO: Consider parsing map structures into equivalent binding representation
|
||||
this.pulsarActivity = pulsarActivity;
|
||||
this.opTpl = optpl;
|
||||
this.cmdTpl = new CommandTemplate(optpl);
|
||||
|
||||
// TODO: At the moment, only supports static "client"
|
||||
String client_name = lookupStaticParameter("client", false, "default");
|
||||
this.clientSpace = pcache.getPulsarSpace(client_name);
|
||||
|
||||
this.opFunc = resolve();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PulsarOp apply(long value) {
|
||||
return opFunc.apply(value);
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolve() {
|
||||
|
||||
String stmtOpType = lookupStaticParameter("optype", true, null);
|
||||
|
||||
if (cmdTpl.containsKey("topic_url")) {
|
||||
throw new PulsarDriverParamException("[resolve()] \"topic_url\" parameter is not valid. Perhaps you mean \"topic_uri\"?");
|
||||
}
|
||||
|
||||
// Doc-level parameter: topic_uri
|
||||
LongFunction<String> topicUriFunc = lookupParameterFunc(PulsarActivityUtil.DOC_LEVEL_PARAMS.TOPIC_URI.label);
|
||||
logger.info("topic_uri: {}", topicUriFunc.apply(0));
|
||||
|
||||
// Doc-level parameter: async_api
|
||||
boolean useAsyncApi = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ASYNC_API.label, false, "false"));
|
||||
LongFunction<Boolean> asyncApiFunc = (l) -> useAsyncApi;
|
||||
logger.info("async_api: {}", useAsyncApi);
|
||||
|
||||
// Doc-level parameter: use_transaction
|
||||
boolean useTransaction = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.USE_TRANSACTION.label, false, "false"));
|
||||
LongFunction<Boolean> useTransactionFunc = (l) -> useTransaction;
|
||||
logger.info("use_transaction: {}", useTransaction);
|
||||
|
||||
// Doc-level parameter: admin_delop
|
||||
boolean adminDelOp = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ADMIN_DELOP.label, false, "false"));
|
||||
LongFunction<Boolean> adminDelOpFunc = (l) -> adminDelOp;
|
||||
logger.info("admin_delop: {}", adminDelOp);
|
||||
|
||||
// Doc-level parameter: seq_tracking
|
||||
boolean seqTracking = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.SEQ_TRACKING.label, false, "false"));
|
||||
LongFunction<Boolean> seqTrackingFunc = (l) -> seqTracking;
|
||||
logger.info("seq_tracking: {}", seqTracking);
|
||||
|
||||
// TODO: Collapse this pattern into a simple version and flatten out all call sites
|
||||
LongFunction<String> payloadRttFieldFunc = lookupParameterFunc(RTT_TRACKING_FIELD, false, "");
|
||||
logger.info("payload_rtt_field_func: {}", payloadRttFieldFunc.apply(0));
|
||||
|
||||
// TODO: Complete implementation for websocket-producer and managed-ledger
|
||||
// Admin operation: create/delete tenant
|
||||
if ( StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TENANT.label) ) {
|
||||
return resolveAdminTenant(clientSpace, asyncApiFunc, adminDelOpFunc);
|
||||
}
|
||||
// Admin operation: create/delete namespace
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_NAMESPACE.label)) {
|
||||
return resolveAdminNamespace(clientSpace, asyncApiFunc, adminDelOpFunc);
|
||||
}
|
||||
// Admin operation: create/delete topic
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TOPIC.label)) {
|
||||
return resolveAdminTopic(clientSpace, topicUriFunc, asyncApiFunc, adminDelOpFunc);
|
||||
}
|
||||
// Regular/non-admin operation: single message sending (producer)
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_SEND.label)) {
|
||||
return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc);
|
||||
}
|
||||
// Regular/non-admin operation: single message consuming from a single topic (consumer)
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_CONSUME.label)) {
|
||||
return resolveMsgConsume(
|
||||
clientSpace,
|
||||
topicUriFunc,
|
||||
asyncApiFunc,
|
||||
useTransactionFunc,
|
||||
seqTrackingFunc,
|
||||
parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE),
|
||||
payloadRttFieldFunc);
|
||||
}
|
||||
// Regular/non-admin operation: single message consuming from multiple-topics (consumer)
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_MULTI_CONSUME.label)) {
|
||||
return resolveMultiTopicMsgConsume(
|
||||
clientSpace,
|
||||
topicUriFunc,
|
||||
asyncApiFunc,
|
||||
useTransactionFunc,
|
||||
seqTrackingFunc,
|
||||
payloadRttFieldFunc);
|
||||
}
|
||||
// Regular/non-admin operation: single message consuming a single topic (reader)
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_READ.label)) {
|
||||
return resolveMsgRead(clientSpace, topicUriFunc, asyncApiFunc);
|
||||
}
|
||||
// // Regular/non-admin operation: batch message processing - batch start
|
||||
// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_START.label)) {
|
||||
// return resolveMsgBatchSendStart(clientSpace, topicUriFunc, asyncApiFunc);
|
||||
// }
|
||||
// // Regular/non-admin operation: batch message processing - message sending (producer)
|
||||
// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND.label)) {
|
||||
// return resolveMsgBatchSend(clientSpace, asyncApiFunc);
|
||||
// }
|
||||
// // Regular/non-admin operation: batch message processing - batch send
|
||||
// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_END.label)) {
|
||||
// return resolveMsgBatchSendEnd(clientSpace, asyncApiFunc);
|
||||
// }
|
||||
// Regular/non-admin operation: end-to-end message processing - sending message
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_SEND.label)) {
|
||||
return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc);
|
||||
}
|
||||
// Regular/non-admin operation: end-to-end message processing - consuming message
|
||||
else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_CONSUME.label)) {
|
||||
return resolveMsgConsume(
|
||||
clientSpace,
|
||||
topicUriFunc,
|
||||
asyncApiFunc,
|
||||
useTransactionFunc,
|
||||
seqTrackingFunc,
|
||||
parseEndToEndStartingTimeSourceParameter(
|
||||
EndToEndStartingTimeSource.MESSAGE_PUBLISH_TIME),
|
||||
payloadRttFieldFunc);
|
||||
}
|
||||
// Invalid operation type
|
||||
else {
|
||||
throw new PulsarDriverUnsupportedOpException();
|
||||
}
|
||||
}
|
||||
|
||||
private EndToEndStartingTimeSource parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource defaultValue) {
|
||||
EndToEndStartingTimeSource endToEndStartingTimeSource = defaultValue;
|
||||
if (cmdTpl.isStatic(PulsarActivityUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label)) {
|
||||
endToEndStartingTimeSource = EndToEndStartingTimeSource.valueOf(cmdTpl.getStatic(PulsarActivityUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label).toUpperCase());
|
||||
}
|
||||
return endToEndStartingTimeSource;
|
||||
}
|
||||
|
||||
// Admin API: create tenant
|
||||
private LongFunction<PulsarOp> resolveAdminTenant(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc)
|
||||
{
|
||||
// "admin_roles" includes comma-separated admin roles:
|
||||
// e.g. role1, role2
|
||||
Set<String> roleSet = lookupStaticParameterSet("admin_roles");
|
||||
LongFunction<Set<String>> adminRolesFunc = (l) -> roleSet;
|
||||
|
||||
// "allowed_cluster" includes comma-separated cluster names:
|
||||
// e.g. cluster1, cluster2
|
||||
Set<String> clusterSet = lookupStaticParameterSet("allowed_clusters");
|
||||
LongFunction<Set<String>> allowedClustersFunc = (l) -> clusterSet;
|
||||
|
||||
LongFunction<String> tenantFunc = lookupParameterFunc("tenant");
|
||||
|
||||
return new PulsarAdminTenantMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
asyncApiFunc,
|
||||
adminDelOpFunc,
|
||||
adminRolesFunc,
|
||||
allowedClustersFunc,
|
||||
tenantFunc);
|
||||
}
|
||||
|
||||
private Set<String> lookupStaticParameterSet(String parameterName) {
|
||||
return Optional.ofNullable(lookupStaticParameter(parameterName))
|
||||
.map(value -> {
|
||||
Set<String> set = Arrays.stream(value.split(","))
|
||||
.map(String::trim)
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
return set;
|
||||
}).orElse(Collections.emptySet());
|
||||
}
|
||||
|
||||
// Admin API: create tenant
|
||||
private LongFunction<PulsarOp> resolveAdminNamespace(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc)
|
||||
{
|
||||
LongFunction<String> namespaceFunc = lookupParameterFunc("namespace");
|
||||
|
||||
return new PulsarAdminNamespaceMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
asyncApiFunc,
|
||||
adminDelOpFunc,
|
||||
namespaceFunc);
|
||||
}
|
||||
|
||||
// Admin API: create partitioned topic
|
||||
private LongFunction<PulsarOp> resolveAdminTopic(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<String> topic_uri_fun,
|
||||
LongFunction<Boolean> asyncApiFunc,
|
||||
LongFunction<Boolean> adminDelOpFunc
|
||||
) {
|
||||
LongFunction<String> enablePartionFunc = lookupParameterFunc("enable_partition");
|
||||
|
||||
LongFunction<String> partitionNumFunc = lookupParameterFunc("partition_num");
|
||||
|
||||
return new PulsarAdminTopicMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
asyncApiFunc,
|
||||
adminDelOpFunc,
|
||||
topic_uri_fun,
|
||||
enablePartionFunc,
|
||||
partitionNumFunc);
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMsgSend(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<String> topic_uri_func,
|
||||
LongFunction<Boolean> async_api_func,
|
||||
LongFunction<Boolean> useTransactionFunc,
|
||||
LongFunction<Boolean> seqTrackingFunc
|
||||
) {
|
||||
LongFunction<Supplier<Transaction>> transactionSupplierFunc =
|
||||
(l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle?
|
||||
|
||||
LongFunction<String> cycle_producer_name_func = lookupParameterFunc("producer_name");
|
||||
|
||||
LongFunction<Producer<?>> producerFunc =
|
||||
(l) -> clientSpace.getProducer(topic_uri_func.apply(l), cycle_producer_name_func.apply(l));
|
||||
|
||||
// check if we're going to simulate producer message out-of-sequence error
|
||||
// - message ordering
|
||||
// - message loss
|
||||
Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> seqErrSimuTypes = parseSimulatedErrorTypes(lookupStaticParameter("seqerr_simu"));
|
||||
|
||||
// message key
|
||||
LongFunction<String> keyFunc = lookupParameterFunc("msg_key");
|
||||
|
||||
// message property
|
||||
LongFunction<String> propFunc = lookupParameterFunc("msg_property");
|
||||
|
||||
LongFunction<String> valueFunc = lookupParameterFunc("msg_value", true);
|
||||
|
||||
return new PulsarProducerMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
async_api_func,
|
||||
useTransactionFunc,
|
||||
seqTrackingFunc,
|
||||
transactionSupplierFunc,
|
||||
producerFunc,
|
||||
seqErrSimuTypes,
|
||||
keyFunc,
|
||||
propFunc,
|
||||
valueFunc);
|
||||
}
|
||||
|
||||
private Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> parseSimulatedErrorTypes(String sequenceErrorSimulatedTypeString) {
|
||||
if (StringUtils.isBlank(sequenceErrorSimulatedTypeString)) {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
return Arrays.stream(StringUtils.split(sequenceErrorSimulatedTypeString, ','))
|
||||
.map(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE::parseSimuType)
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMsgConsume(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<String> topic_uri_func,
|
||||
LongFunction<Boolean> async_api_func,
|
||||
LongFunction<Boolean> useTransactionFunc,
|
||||
LongFunction<Boolean> seqTrackingFunc,
|
||||
EndToEndStartingTimeSource endToEndStartingTimeSource,
|
||||
LongFunction<String> rttTrackingFieldFunc
|
||||
) {
|
||||
LongFunction<String> subscription_name_func = lookupParameterFunc("subscription_name");
|
||||
|
||||
LongFunction<String> subscription_type_func = lookupParameterFunc("subscription_type");
|
||||
|
||||
LongFunction<String> consumer_name_func = lookupParameterFunc("consumer_name");
|
||||
|
||||
LongFunction<String> ranges_func = lookupParameterFunc("ranges", false, "");
|
||||
|
||||
LongFunction<Supplier<Transaction>> transactionSupplierFunc =
|
||||
(l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle?
|
||||
|
||||
LongFunction<Consumer<?>> consumerFunc = (l) ->
|
||||
clientSpace.getConsumer(
|
||||
topic_uri_func.apply(l),
|
||||
subscription_name_func.apply(l),
|
||||
subscription_type_func.apply(l),
|
||||
consumer_name_func.apply(l),
|
||||
ranges_func.apply(l)
|
||||
);
|
||||
|
||||
return new PulsarConsumerMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
async_api_func,
|
||||
useTransactionFunc,
|
||||
seqTrackingFunc,
|
||||
transactionSupplierFunc,
|
||||
consumerFunc,
|
||||
endToEndStartingTimeSource,
|
||||
rttTrackingFieldFunc);
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMultiTopicMsgConsume(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<String> topic_uri_func,
|
||||
LongFunction<Boolean> async_api_func,
|
||||
LongFunction<Boolean> useTransactionFunc,
|
||||
LongFunction<Boolean> seqTrackingFunc,
|
||||
LongFunction<String> payloadRttFieldFunc
|
||||
) {
|
||||
// Topic list (multi-topic)
|
||||
LongFunction<String> topic_names_func = lookupParameterFunc("topic_names");
|
||||
|
||||
// Topic pattern (multi-topic)
|
||||
LongFunction<String> topics_pattern_func = lookupParameterFunc("topics_pattern");
|
||||
|
||||
LongFunction<String> subscription_name_func = lookupParameterFunc("subscription_name");
|
||||
|
||||
LongFunction<String> subscription_type_func = lookupParameterFunc("subscription_type");
|
||||
|
||||
LongFunction<String> consumer_name_func = lookupParameterFunc("consumer_name");
|
||||
|
||||
LongFunction<Supplier<Transaction>> transactionSupplierFunc =
|
||||
(l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle?
|
||||
|
||||
LongFunction<Consumer<?>> mtConsumerFunc = (l) ->
|
||||
clientSpace.getMultiTopicConsumer(
|
||||
topic_uri_func.apply(l),
|
||||
topic_names_func.apply(l),
|
||||
topics_pattern_func.apply(l),
|
||||
subscription_name_func.apply(l),
|
||||
subscription_type_func.apply(l),
|
||||
consumer_name_func.apply(l)
|
||||
);
|
||||
|
||||
return new PulsarConsumerMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
async_api_func,
|
||||
useTransactionFunc,
|
||||
seqTrackingFunc,
|
||||
transactionSupplierFunc,
|
||||
mtConsumerFunc,
|
||||
parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE),
|
||||
payloadRttFieldFunc);
|
||||
}
|
||||
|
||||
private LongFunction<String> lookupParameterFunc(String parameterName) {
|
||||
return lookupParameterFunc(parameterName, false, null);
|
||||
}
|
||||
|
||||
private LongFunction<String> lookupParameterFunc(String parameterName, boolean required) {
|
||||
return lookupParameterFunc(parameterName, required, null);
|
||||
}
|
||||
|
||||
private LongFunction<String> lookupParameterFunc(String parameterName, boolean required, String defaultValue) {
|
||||
if (cmdTpl.containsKey(parameterName)) {
|
||||
LongFunction<String> lookupFunc;
|
||||
if (cmdTpl.isStatic(parameterName)) {
|
||||
String staticValue = cmdTpl.getStatic(parameterName);
|
||||
lookupFunc = (l) -> staticValue;
|
||||
} else if (cmdTpl.isDynamic(parameterName)) {
|
||||
lookupFunc = (l) -> cmdTpl.getDynamic(parameterName, l);
|
||||
} else {
|
||||
lookupFunc = (l) -> defaultValue;
|
||||
}
|
||||
return lookupFunc;
|
||||
} else {
|
||||
if (required) {
|
||||
throw new PulsarDriverParamException("\"" + parameterName + "\" field must be specified!");
|
||||
} else {
|
||||
return (l) -> defaultValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String lookupStaticParameter(String parameterName) {
|
||||
return lookupStaticParameter(parameterName, false, null);
|
||||
}
|
||||
|
||||
private String lookupStaticParameter(String parameterName, boolean required, String defaultValue) {
|
||||
if (cmdTpl.containsKey(parameterName)) {
|
||||
if (cmdTpl.isStatic(parameterName)) {
|
||||
return cmdTpl.getStatic(parameterName);
|
||||
} else if (cmdTpl.isDynamic(parameterName)) {
|
||||
throw new PulsarDriverParamException("\"" + parameterName + "\" parameter must be static");
|
||||
} else {
|
||||
return defaultValue;
|
||||
}
|
||||
} else {
|
||||
if (required) {
|
||||
throw new PulsarDriverParamException("\"" + parameterName + "\" field must be specified!");
|
||||
} else {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private LongFunction<Boolean> toBooleanFunc(LongFunction<String> parameterFunc) {
|
||||
return (l) -> BooleanUtils.toBoolean(parameterFunc.apply(l));
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMsgRead(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<String> topic_uri_func,
|
||||
LongFunction<Boolean> async_api_func
|
||||
) {
|
||||
LongFunction<String> reader_name_func = lookupParameterFunc("reader_name");
|
||||
|
||||
LongFunction<String> start_msg_pos_str_func = lookupParameterFunc("start_msg_position");
|
||||
|
||||
LongFunction<Reader<?>> readerFunc = (l) ->
|
||||
clientSpace.getReader(
|
||||
topic_uri_func.apply(l),
|
||||
reader_name_func.apply(l),
|
||||
start_msg_pos_str_func.apply(l)
|
||||
);
|
||||
|
||||
return new PulsarReaderMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
async_api_func,
|
||||
readerFunc);
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMsgBatchSendStart(
|
||||
PulsarSpace clientSpace,
|
||||
LongFunction<String> topic_uri_func,
|
||||
LongFunction<Boolean> asyncApiFunc)
|
||||
{
|
||||
LongFunction<String> cycle_batch_producer_name_func = lookupParameterFunc("batch_producer_name");
|
||||
|
||||
LongFunction<Producer<?>> batchProducerFunc =
|
||||
(l) -> clientSpace.getProducer(topic_uri_func.apply(l), cycle_batch_producer_name_func.apply(l));
|
||||
|
||||
return new PulsarBatchProducerStartMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
asyncApiFunc,
|
||||
batchProducerFunc);
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMsgBatchSend(PulsarSpace clientSpace,
|
||||
LongFunction<Boolean> asyncApiFunc)
|
||||
{
|
||||
LongFunction<String> keyFunc = lookupParameterFunc("msg_key");
|
||||
|
||||
// message property
|
||||
LongFunction<String> propFunc = lookupParameterFunc("msg_property");
|
||||
|
||||
LongFunction<String> valueFunc = lookupParameterFunc("msg_value", true);
|
||||
|
||||
return new PulsarBatchProducerMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
asyncApiFunc,
|
||||
keyFunc,
|
||||
propFunc,
|
||||
valueFunc);
|
||||
}
|
||||
|
||||
private LongFunction<PulsarOp> resolveMsgBatchSendEnd(PulsarSpace clientSpace,
|
||||
LongFunction<Boolean> asyncApiFunc)
|
||||
{
|
||||
return new PulsarBatchProducerEndMapper(
|
||||
cmdTpl,
|
||||
clientSpace,
|
||||
pulsarActivity,
|
||||
asyncApiFunc);
|
||||
}
|
||||
}
|
@ -1,166 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
import java.util.Iterator;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/**
|
||||
* Detects message loss, message duplication and out-of-order message delivery
|
||||
* based on a monotonic sequence number that each received message contains.
|
||||
* <p>
|
||||
* Out-of-order messages are detected with a maximum look behind of 1000 sequence number entries.
|
||||
* This is currently defined as a constant, {@link ReceivedMessageSequenceTracker#DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS}.
|
||||
*/
|
||||
class ReceivedMessageSequenceTracker implements AutoCloseable {
|
||||
private static final int DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS = 1000;
|
||||
private static final int DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS = 1000;
|
||||
// message out-of-sequence error counter
|
||||
private final Counter msgErrOutOfSeqCounter;
|
||||
// duplicate message error counter
|
||||
private final Counter msgErrDuplicateCounter;
|
||||
// message loss error counter
|
||||
private final Counter msgErrLossCounter;
|
||||
private final SortedSet<Long> pendingOutOfSeqNumbers;
|
||||
private final int maxTrackOutOfOrderSequenceNumbers;
|
||||
private final SortedSet<Long> skippedSeqNumbers;
|
||||
private final int maxTrackSkippedSequenceNumbers;
|
||||
private long expectedNumber = -1;
|
||||
|
||||
public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter) {
|
||||
this(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter,
|
||||
DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS, DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS);
|
||||
}
|
||||
|
||||
public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter,
|
||||
int maxTrackOutOfOrderSequenceNumbers, int maxTrackSkippedSequenceNumbers) {
|
||||
this.msgErrOutOfSeqCounter = msgErrOutOfSeqCounter;
|
||||
this.msgErrDuplicateCounter = msgErrDuplicateCounter;
|
||||
this.msgErrLossCounter = msgErrLossCounter;
|
||||
this.maxTrackOutOfOrderSequenceNumbers = maxTrackOutOfOrderSequenceNumbers;
|
||||
this.maxTrackSkippedSequenceNumbers = maxTrackSkippedSequenceNumbers;
|
||||
this.pendingOutOfSeqNumbers = new TreeSet<>();
|
||||
this.skippedSeqNumbers = new TreeSet<>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Notifies the tracker about a received sequence number
|
||||
*
|
||||
* @param sequenceNumber the sequence number of the received message
|
||||
*/
|
||||
public void sequenceNumberReceived(long sequenceNumber) {
|
||||
if (expectedNumber == -1) {
|
||||
expectedNumber = sequenceNumber + 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (sequenceNumber < expectedNumber) {
|
||||
if (skippedSeqNumbers.remove(sequenceNumber)) {
|
||||
// late out-of-order delivery was detected
|
||||
// decrease the loss counter
|
||||
msgErrLossCounter.dec();
|
||||
// increment the out-of-order counter
|
||||
msgErrOutOfSeqCounter.inc();
|
||||
} else {
|
||||
msgErrDuplicateCounter.inc();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
boolean messagesSkipped = false;
|
||||
if (sequenceNumber > expectedNumber) {
|
||||
if (pendingOutOfSeqNumbers.size() == maxTrackOutOfOrderSequenceNumbers) {
|
||||
messagesSkipped = processLowestPendingOutOfSequenceNumber();
|
||||
}
|
||||
if (!pendingOutOfSeqNumbers.add(sequenceNumber)) {
|
||||
msgErrDuplicateCounter.inc();
|
||||
}
|
||||
} else {
|
||||
// sequenceNumber == expectedNumber
|
||||
expectedNumber++;
|
||||
}
|
||||
processPendingOutOfSequenceNumbers(messagesSkipped);
|
||||
cleanUpTooFarBehindOutOfSequenceNumbers();
|
||||
}
|
||||
|
||||
private boolean processLowestPendingOutOfSequenceNumber() {
|
||||
// remove the lowest pending out of sequence number
|
||||
Long lowestOutOfSeqNumber = pendingOutOfSeqNumbers.first();
|
||||
pendingOutOfSeqNumbers.remove(lowestOutOfSeqNumber);
|
||||
if (lowestOutOfSeqNumber > expectedNumber) {
|
||||
// skip the expected number ahead to the number after the lowest sequence number
|
||||
// increment the counter with the amount of sequence numbers that got skipped
|
||||
// keep track of the skipped sequence numbers to detect late out-of-order message delivery
|
||||
for (long l = expectedNumber; l < lowestOutOfSeqNumber; l++) {
|
||||
msgErrLossCounter.inc();
|
||||
skippedSeqNumbers.add(l);
|
||||
if (skippedSeqNumbers.size() > maxTrackSkippedSequenceNumbers) {
|
||||
skippedSeqNumbers.remove(skippedSeqNumbers.first());
|
||||
}
|
||||
}
|
||||
expectedNumber = lowestOutOfSeqNumber + 1;
|
||||
return true;
|
||||
} else {
|
||||
msgErrLossCounter.inc();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void processPendingOutOfSequenceNumbers(boolean messagesSkipped) {
|
||||
// check if there are previously received out-of-order sequence number that have been received
|
||||
while (pendingOutOfSeqNumbers.remove(expectedNumber)) {
|
||||
expectedNumber++;
|
||||
if (!messagesSkipped) {
|
||||
msgErrOutOfSeqCounter.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanUpTooFarBehindOutOfSequenceNumbers() {
|
||||
// remove sequence numbers that are too far behind
|
||||
for (Iterator<Long> iterator = pendingOutOfSeqNumbers.iterator(); iterator.hasNext(); ) {
|
||||
Long number = iterator.next();
|
||||
if (number < expectedNumber - maxTrackOutOfOrderSequenceNumbers) {
|
||||
msgErrLossCounter.inc();
|
||||
iterator.remove();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles the possible pending out of sequence numbers. Mainly needed in unit tests to assert the
|
||||
* counter values.
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
while (!pendingOutOfSeqNumbers.isEmpty()) {
|
||||
processPendingOutOfSequenceNumbers(processLowestPendingOutOfSequenceNumber());
|
||||
}
|
||||
}
|
||||
|
||||
public int getMaxTrackOutOfOrderSequenceNumbers() {
|
||||
return maxTrackOutOfOrderSequenceNumbers;
|
||||
}
|
||||
|
||||
public int getMaxTrackSkippedSequenceNumbers() {
|
||||
return maxTrackSkippedSequenceNumbers;
|
||||
}
|
||||
}
|
@ -1,33 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
/**
|
||||
* Base type of all Sync Pulsar Operations including Producers and Consumers.
|
||||
*/
|
||||
public abstract class SyncPulsarOp implements PulsarOp {
|
||||
|
||||
public void run(Runnable timeTracker) {
|
||||
try {
|
||||
this.run();
|
||||
} finally {
|
||||
timeTracker.run();
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void run();
|
||||
}
|
@ -1,123 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.util;
|
||||
|
||||
import org.apache.avro.io.DecoderFactory;
|
||||
import org.apache.avro.io.JsonDecoder;
|
||||
import org.apache.avro.io.BinaryDecoder;
|
||||
import org.apache.pulsar.client.api.schema.Field;
|
||||
import org.apache.pulsar.client.api.schema.GenericRecord;
|
||||
import org.apache.pulsar.client.api.schema.GenericRecordBuilder;
|
||||
import org.apache.pulsar.client.impl.schema.SchemaInfoImpl;
|
||||
import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema;
|
||||
import org.apache.pulsar.common.schema.SchemaInfo;
|
||||
import org.apache.pulsar.common.schema.SchemaType;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
public class AvroUtil {
|
||||
////////////////////////
|
||||
// Get an OSS Apache Avro schema from a string definition
|
||||
public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) {
|
||||
return new org.apache.avro.Schema.Parser().parse(avroSchemDef);
|
||||
}
|
||||
|
||||
// Get an OSS Apache Avro schema record from a JSON string that matches a specific OSS Apache Avro schema
|
||||
public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, String jsonData) {
|
||||
org.apache.avro.generic.GenericRecord record = null;
|
||||
|
||||
try {
|
||||
org.apache.avro.generic.GenericDatumReader<org.apache.avro.generic.GenericData.Record> reader;
|
||||
reader = new org.apache.avro.generic.GenericDatumReader<>(schema);
|
||||
|
||||
JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, jsonData);
|
||||
|
||||
record = reader.read(null, decoder);
|
||||
} catch (IOException ioe) {
|
||||
ioe.printStackTrace();
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
// Get an OSS Apache Avro schema record from a byte array that matches a specific OSS Apache Avro schema
|
||||
public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, byte[] bytesData) {
|
||||
org.apache.avro.generic.GenericRecord record = null;
|
||||
|
||||
try {
|
||||
org.apache.avro.generic.GenericDatumReader<org.apache.avro.generic.GenericData.Record> reader;
|
||||
reader = new org.apache.avro.generic.GenericDatumReader<>(schema);
|
||||
|
||||
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytesData, null);
|
||||
|
||||
record = reader.read(null, decoder);
|
||||
} catch (IOException ioe) {
|
||||
ioe.printStackTrace();
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
|
||||
////////////////////////
|
||||
// Get a Pulsar Avro schema from a string definition
|
||||
public static GenericAvroSchema GetSchema_PulsarAvro(String schemaName, String avroSchemDef) {
|
||||
SchemaInfo schemaInfo = SchemaInfoImpl.builder()
|
||||
.schema(avroSchemDef.getBytes(StandardCharsets.UTF_8))
|
||||
.type(SchemaType.AVRO)
|
||||
.properties(new HashMap<>())
|
||||
.name(schemaName)
|
||||
.build();
|
||||
return new GenericAvroSchema(schemaInfo);
|
||||
}
|
||||
|
||||
// Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema
|
||||
public static GenericRecord GetGenericRecord_PulsarAvro(
|
||||
GenericAvroSchema pulsarGenericAvroSchema,
|
||||
org.apache.avro.generic.GenericRecord apacheAvroGenericRecord)
|
||||
{
|
||||
GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder();
|
||||
|
||||
List<Field> fieldList = pulsarGenericAvroSchema.getFields();
|
||||
for (Field field : fieldList) {
|
||||
String fieldName = field.getName();
|
||||
recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName));
|
||||
}
|
||||
|
||||
return recordBuilder.build();
|
||||
}
|
||||
|
||||
// Get a Pulsar Avro record (GenericRecord) from a JSON string that matches a specific Pulsar Avro schema
|
||||
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, String avroSchemDefStr, String jsonData) {
|
||||
org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr);
|
||||
return GetGenericRecord_PulsarAvro(genericAvroSchema, avroSchema, jsonData);
|
||||
}
|
||||
|
||||
public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, org.apache.avro.Schema avroSchema, String jsonData) {
|
||||
org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData);
|
||||
return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord);
|
||||
}
|
||||
public static GenericRecord GetGenericRecord_PulsarAvro(String schemaName, String avroSchemDefStr, String jsonData) {
|
||||
GenericAvroSchema genericAvroSchema = GetSchema_PulsarAvro(schemaName, avroSchemDefStr);
|
||||
org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr);
|
||||
org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData);
|
||||
|
||||
return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord);
|
||||
}
|
||||
}
|
@ -1,535 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.util;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.pulsar.client.api.Schema;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class PulsarActivityUtil {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarActivityUtil.class);
|
||||
|
||||
// Supported message operation types
|
||||
// TODO: websocket-producer and managed-ledger
|
||||
public enum OP_TYPES {
|
||||
ADMIN_TENANT("admin-tenant"),
|
||||
ADMIN_NAMESPACE("admin-namespace"),
|
||||
ADMIN_TOPIC("admin-topic"),
|
||||
E2E_MSG_PROC_SEND("e22-msg-proc-send"),
|
||||
E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"),
|
||||
// BATCH_MSG_SEND_START("batch-msg-send-start"),
|
||||
// BATCH_MSG_SEND("batch-msg-send"),
|
||||
// BATCH_MSG_SEND_END("batch-msg-send-end"),
|
||||
MSG_SEND("msg-send"),
|
||||
MSG_CONSUME("msg-consume"),
|
||||
MSG_READ("msg-read"),
|
||||
MSG_MULTI_CONSUME("msg-mt-consume");
|
||||
|
||||
public final String label;
|
||||
|
||||
OP_TYPES(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static boolean isValidClientType(String type) {
|
||||
return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type));
|
||||
}
|
||||
|
||||
public static final String MSG_SEQUENCE_NUMBER = "sequence_number";
|
||||
|
||||
///////
|
||||
// Valid document level parameters for Pulsar NB yaml file
|
||||
public enum DOC_LEVEL_PARAMS {
|
||||
TOPIC_URI("topic_uri"),
|
||||
ASYNC_API("async_api"),
|
||||
USE_TRANSACTION("use_transaction"),
|
||||
ADMIN_DELOP("admin_delop"),
|
||||
SEQ_TRACKING("seq_tracking"),
|
||||
MSG_DEDUP_BROKER("msg_dedup_broker"),
|
||||
E2E_STARTING_TIME_SOURCE("e2e_starting_time_source");
|
||||
|
||||
public final String label;
|
||||
|
||||
DOC_LEVEL_PARAMS(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isValidDocLevelParam(String param) {
|
||||
return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param));
|
||||
}
|
||||
|
||||
///////
|
||||
// Valid Pulsar API type
|
||||
public enum PULSAR_API_TYPE {
|
||||
PRODUCER("producer"),
|
||||
CONSUMER("consumer"),
|
||||
READER("reader");
|
||||
|
||||
public final String label;
|
||||
|
||||
PULSAR_API_TYPE(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isValidPulsarApiType(String param) {
|
||||
return Arrays.stream(PULSAR_API_TYPE.values()).anyMatch(t -> t.label.equals(param));
|
||||
}
|
||||
public static String getValidPulsarApiTypeList() {
|
||||
return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
///////
|
||||
// Valid persistence type
|
||||
public enum PERSISTENT_TYPES {
|
||||
PERSISTENT("persistent"),
|
||||
NON_PERSISTENT("non-persistent")
|
||||
;
|
||||
|
||||
public final String label;
|
||||
PERSISTENT_TYPES(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isValidPersistenceType(String type) {
|
||||
return Arrays.stream(PERSISTENT_TYPES.values()).anyMatch(t -> t.label.equals(type));
|
||||
}
|
||||
|
||||
///////
|
||||
// Valid Pulsar client configuration (activity-level settings)
|
||||
// - https://pulsar.apache.org/docs/en/client-libraries-java/#client
|
||||
public enum CLNT_CONF_KEY {
|
||||
serviceUrl("serviceUrl"),
|
||||
authPulginClassName("authPluginClassName"),
|
||||
authParams("authParams"),
|
||||
pperationTimeoutMs("operationTimeoutMs"),
|
||||
statsIntervalSeconds("statsIntervalSeconds"),
|
||||
numIoThreads("numIoThreads"),
|
||||
numListenerThreads("numListenerThreads"),
|
||||
useTcpNoDelay("useTcpNoDelay"),
|
||||
useTls("useTls"),
|
||||
tlsTrustCertsFilePath("tlsTrustCertsFilePath"),
|
||||
tlsAllowInsecureConnection("tlsAllowInsecureConnection"),
|
||||
tlsHostnameVerificationEnable("tlsHostnameVerificationEnable"),
|
||||
concurrentLookupRequest("concurrentLookupRequest"),
|
||||
maxLookupRequest("maxLookupRequest"),
|
||||
maxNumberOfRejectedRequestPerConnection("maxNumberOfRejectedRequestPerConnection"),
|
||||
keepAliveIntervalSeconds("keepAliveIntervalSeconds"),
|
||||
connectionTimeoutMs("connectionTimeoutMs"),
|
||||
requestTimeoutMs("requestTimeoutMs"),
|
||||
defaultBackoffIntervalNanos("defaultBackoffIntervalNanos"),
|
||||
maxBackoffIntervalNanos("maxBackoffIntervalNanos")
|
||||
;
|
||||
|
||||
public final String label;
|
||||
CLNT_CONF_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isValidClientConfItem(String item) {
|
||||
return Arrays.stream(CLNT_CONF_KEY.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Standard producer configuration (activity-level settings)
|
||||
// - https://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
|
||||
public enum PRODUCER_CONF_STD_KEY {
|
||||
topicName("topicName"),
|
||||
producerName("producerName"),
|
||||
sendTimeoutMs("sendTimeoutMs"),
|
||||
blockIfQueueFull("blockIfQueueFull"),
|
||||
maxPendingMessages("maxPendingMessages"),
|
||||
maxPendingMessagesAcrossPartitions("maxPendingMessagesAcrossPartitions"),
|
||||
messageRoutingMode("messageRoutingMode"),
|
||||
hashingScheme("hashingScheme"),
|
||||
cryptoFailureAction("cryptoFailureAction"),
|
||||
batchingMaxPublishDelayMicros("batchingMaxPublishDelayMicros"),
|
||||
batchingMaxMessages("batchingMaxMessages"),
|
||||
batchingEnabled("batchingEnabled"),
|
||||
compressionType("compressionType");
|
||||
|
||||
public final String label;
|
||||
|
||||
PRODUCER_CONF_STD_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isStandardProducerConfItem(String item) {
|
||||
return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Standard consumer configuration (activity-level settings)
|
||||
// - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer
|
||||
public enum CONSUMER_CONF_STD_KEY {
|
||||
topicNames("topicNames"),
|
||||
topicsPattern("topicsPattern"),
|
||||
subscriptionName("subscriptionName"),
|
||||
subscriptionType("subscriptionType"),
|
||||
receiverQueueSize("receiverQueueSize"),
|
||||
acknowledgementsGroupTimeMicros("acknowledgementsGroupTimeMicros"),
|
||||
negativeAckRedeliveryDelayMicros("negativeAckRedeliveryDelayMicros"),
|
||||
maxTotalReceiverQueueSizeAcrossPartitions("maxTotalReceiverQueueSizeAcrossPartitions"),
|
||||
consumerName("consumerName"),
|
||||
ackTimeoutMillis("ackTimeoutMillis"),
|
||||
tickDurationMillis("tickDurationMillis"),
|
||||
priorityLevel("priorityLevel"),
|
||||
cryptoFailureAction("cryptoFailureAction"),
|
||||
properties("properties"),
|
||||
readCompacted("readCompacted"),
|
||||
subscriptionInitialPosition("subscriptionInitialPosition"),
|
||||
patternAutoDiscoveryPeriod("patternAutoDiscoveryPeriod"),
|
||||
regexSubscriptionMode("regexSubscriptionMode"),
|
||||
deadLetterPolicy("deadLetterPolicy"),
|
||||
autoUpdatePartitions("autoUpdatePartitions"),
|
||||
replicateSubscriptionState("replicateSubscriptionState");
|
||||
|
||||
public final String label;
|
||||
|
||||
CONSUMER_CONF_STD_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isStandardConsumerConfItem(String item) {
|
||||
return Arrays.stream(CONSUMER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Custom consumer configuration (activity-level settings)
|
||||
// - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer
|
||||
// - NB Pulsar driver consumer operation specific
|
||||
public enum CONSUMER_CONF_CUSTOM_KEY {
|
||||
timeout("timeout");
|
||||
|
||||
public final String label;
|
||||
|
||||
CONSUMER_CONF_CUSTOM_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isCustomConsumerConfItem(String item) {
|
||||
return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Pulsar subscription type
|
||||
public enum SUBSCRIPTION_TYPE {
|
||||
Exclusive("Exclusive"),
|
||||
Failover("Failover"),
|
||||
Shared("Shared"),
|
||||
Key_Shared("Key_Shared");
|
||||
|
||||
public final String label;
|
||||
|
||||
SUBSCRIPTION_TYPE(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isValidSubscriptionType(String item) {
|
||||
return Arrays.stream(SUBSCRIPTION_TYPE.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
public static String getValidSubscriptionTypeList() {
|
||||
return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
///////
|
||||
// Standard reader configuration (activity-level settings)
|
||||
// - https://pulsar.apache.org/docs/en/client-libraries-java/#reader
|
||||
public enum READER_CONF_STD_KEY {
|
||||
topicName("topicName"),
|
||||
receiverQueueSize("receiverQueueSize"),
|
||||
readerListener("readerListener"),
|
||||
readerName("readerName"),
|
||||
subscriptionRolePrefix("subscriptionRolePrefix"),
|
||||
cryptoKeyReader("cryptoKeyReader"),
|
||||
cryptoFailureAction("cryptoFailureAction"),
|
||||
readCompacted("readCompacted"),
|
||||
resetIncludeHead("resetIncludeHead");
|
||||
|
||||
public final String label;
|
||||
|
||||
READER_CONF_STD_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isStandardReaderConfItem(String item) {
|
||||
return Arrays.stream(READER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Custom reader configuration (activity-level settings)
|
||||
// - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#reader
|
||||
// - NB Pulsar driver reader operation specific
|
||||
public enum READER_CONF_CUSTOM_KEY {
|
||||
startMessagePos("startMessagePos");
|
||||
|
||||
public final String label;
|
||||
|
||||
READER_CONF_CUSTOM_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isCustomReaderConfItem(String item) {
|
||||
return Arrays.stream(READER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Valid read positions for a Pulsar reader
|
||||
public enum READER_MSG_POSITION_TYPE {
|
||||
earliest("earliest"),
|
||||
latest("latest"),
|
||||
custom("custom");
|
||||
|
||||
public final String label;
|
||||
|
||||
READER_MSG_POSITION_TYPE(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
public static boolean isValideReaderStartPosition(String item) {
|
||||
return Arrays.stream(READER_MSG_POSITION_TYPE.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
|
||||
///////
|
||||
// Pulsar subscription type
|
||||
public enum SEQ_ERROR_SIMU_TYPE {
|
||||
OutOfOrder("out_of_order"),
|
||||
MsgLoss("msg_loss"),
|
||||
MsgDup("msg_dup");
|
||||
|
||||
public final String label;
|
||||
|
||||
SEQ_ERROR_SIMU_TYPE(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
private static final Map<String, SEQ_ERROR_SIMU_TYPE> MAPPING = new HashMap<>();
|
||||
|
||||
static {
|
||||
for (SEQ_ERROR_SIMU_TYPE simuType : values()) {
|
||||
MAPPING.put(simuType.label, simuType);
|
||||
MAPPING.put(simuType.label.toLowerCase(), simuType);
|
||||
MAPPING.put(simuType.label.toUpperCase(), simuType);
|
||||
MAPPING.put(simuType.name(), simuType);
|
||||
MAPPING.put(simuType.name().toLowerCase(), simuType);
|
||||
MAPPING.put(simuType.name().toUpperCase(), simuType);
|
||||
}
|
||||
}
|
||||
|
||||
public static Optional<SEQ_ERROR_SIMU_TYPE> parseSimuType(String simuTypeString) {
|
||||
return Optional.ofNullable(MAPPING.get(simuTypeString.trim()));
|
||||
}
|
||||
}
|
||||
public static boolean isValidSeqErrSimuType(String item) {
|
||||
return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item));
|
||||
}
|
||||
public static String getValidSeqErrSimuTypeList() {
|
||||
return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
///////
|
||||
// Valid websocket-producer configuration (activity-level settings)
|
||||
// TODO: to be added
|
||||
public enum WEBSKT_PRODUCER_CONF_KEY {
|
||||
;
|
||||
|
||||
public final String label;
|
||||
|
||||
WEBSKT_PRODUCER_CONF_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
|
||||
///////
|
||||
// Valid managed-ledger configuration (activity-level settings)
|
||||
// TODO: to be added
|
||||
public enum MANAGED_LEDGER_CONF_KEY {
|
||||
;
|
||||
|
||||
public final String label;
|
||||
MANAGED_LEDGER_CONF_KEY(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
|
||||
///////
|
||||
// Primitive Schema type
|
||||
public static boolean isPrimitiveSchemaTypeStr(String typeStr) {
|
||||
boolean isPrimitive = false;
|
||||
|
||||
// Use "BYTES" as the default type if the type string is not explicitly specified
|
||||
if (StringUtils.isBlank(typeStr)) {
|
||||
typeStr = "BYTES";
|
||||
}
|
||||
|
||||
if (typeStr.equalsIgnoreCase("BOOLEAN") || typeStr.equalsIgnoreCase("INT8") ||
|
||||
typeStr.equalsIgnoreCase("INT16") || typeStr.equalsIgnoreCase("INT32") ||
|
||||
typeStr.equalsIgnoreCase("INT64") || typeStr.equalsIgnoreCase("FLOAT") ||
|
||||
typeStr.equalsIgnoreCase("DOUBLE") || typeStr.equalsIgnoreCase("BYTES") ||
|
||||
typeStr.equalsIgnoreCase("DATE") || typeStr.equalsIgnoreCase("TIME") ||
|
||||
typeStr.equalsIgnoreCase("TIMESTAMP") || typeStr.equalsIgnoreCase("INSTANT") ||
|
||||
typeStr.equalsIgnoreCase("LOCAL_DATE") || typeStr.equalsIgnoreCase("LOCAL_TIME") ||
|
||||
typeStr.equalsIgnoreCase("LOCAL_DATE_TIME")) {
|
||||
isPrimitive = true;
|
||||
}
|
||||
|
||||
return isPrimitive;
|
||||
}
|
||||
public static Schema<?> getPrimitiveTypeSchema(String typeStr) {
|
||||
Schema<?> schema;
|
||||
|
||||
switch (typeStr.toUpperCase()) {
|
||||
case "BOOLEAN":
|
||||
schema = Schema.BOOL;
|
||||
break;
|
||||
case "INT8":
|
||||
schema = Schema.INT8;
|
||||
break;
|
||||
case "INT16":
|
||||
schema = Schema.INT16;
|
||||
break;
|
||||
case "INT32":
|
||||
schema = Schema.INT32;
|
||||
break;
|
||||
case "INT64":
|
||||
schema = Schema.INT64;
|
||||
break;
|
||||
case "FLOAT":
|
||||
schema = Schema.FLOAT;
|
||||
break;
|
||||
case "DOUBLE":
|
||||
schema = Schema.DOUBLE;
|
||||
break;
|
||||
case "DATE":
|
||||
schema = Schema.DATE;
|
||||
break;
|
||||
case "TIME":
|
||||
schema = Schema.TIME;
|
||||
break;
|
||||
case "TIMESTAMP":
|
||||
schema = Schema.TIMESTAMP;
|
||||
break;
|
||||
case "INSTANT":
|
||||
schema = Schema.INSTANT;
|
||||
break;
|
||||
case "LOCAL_DATE":
|
||||
schema = Schema.LOCAL_DATE;
|
||||
break;
|
||||
case "LOCAL_TIME":
|
||||
schema = Schema.LOCAL_TIME;
|
||||
break;
|
||||
case "LOCAL_DATE_TIME":
|
||||
schema = Schema.LOCAL_DATE_TIME;
|
||||
break;
|
||||
// Use BYTES as the default schema type if the type string is not specified
|
||||
case "":
|
||||
case "BYTES":
|
||||
schema = Schema.BYTES;
|
||||
break;
|
||||
// Report an error if non-valid, non-empty schema type string is provided
|
||||
default:
|
||||
throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr);
|
||||
}
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
///////
|
||||
// Complex strut type: Avro or Json
|
||||
public static boolean isAvroSchemaTypeStr(String typeStr) {
|
||||
return typeStr.equalsIgnoreCase("AVRO");
|
||||
}
|
||||
public static boolean isKeyValueTypeStr(String typeStr) {
|
||||
return typeStr.equalsIgnoreCase("KEY_VALUE");
|
||||
}
|
||||
|
||||
// automatic decode the type from the Registry
|
||||
public static boolean isAutoConsumeSchemaTypeStr(String typeStr) {
|
||||
return typeStr.equalsIgnoreCase("AUTO_CONSUME");
|
||||
}
|
||||
public static Schema<?> getAvroSchema(String typeStr, String definitionStr) {
|
||||
String schemaDefinitionStr = definitionStr;
|
||||
String filePrefix = "file://";
|
||||
Schema<?> schema;
|
||||
|
||||
// Check if payloadStr points to a file (e.g. "file:///path/to/a/file")
|
||||
if (isAvroSchemaTypeStr(typeStr)) {
|
||||
if (StringUtils.isBlank(schemaDefinitionStr)) {
|
||||
throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!");
|
||||
} else if (schemaDefinitionStr.startsWith(filePrefix)) {
|
||||
try {
|
||||
Path filePath = Paths.get(URI.create(schemaDefinitionStr));
|
||||
schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII);
|
||||
} catch (IOException ioe) {
|
||||
throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr);
|
||||
} else {
|
||||
throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr);
|
||||
}
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
///////
|
||||
// Generate effective key string
|
||||
public static String buildCacheKey(String... keyParts) {
|
||||
// Ignore blank keyPart
|
||||
String joinedKeyStr =
|
||||
Stream.of(keyParts)
|
||||
.filter(s -> !StringUtils.isBlank(s))
|
||||
.collect(Collectors.joining(","));
|
||||
|
||||
return Base64.getEncoder().encodeToString(joinedKeyStr.getBytes());
|
||||
}
|
||||
|
||||
///////
|
||||
// Convert JSON string to a key/value map
|
||||
public static Map<String, String> convertJsonToMap(String jsonStr) throws Exception {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
return mapper.readValue(jsonStr, Map.class);
|
||||
}
|
||||
|
||||
///////
|
||||
// Get full namespace name (<tenant>/<namespace>) from a Pulsar topic URI
|
||||
public static String getFullNamespaceName(String topicUri) {
|
||||
// Get tenant/namespace string
|
||||
// - topicUri : persistent://<tenant>/<namespace>/<topic>
|
||||
// - tmpStr : <tenant>/<namespace>/<topic>
|
||||
// - fullNsName : <tenant>/<namespace>
|
||||
|
||||
String tmpStr = StringUtils.substringAfter(topicUri,"://");
|
||||
return StringUtils.substringBeforeLast(tmpStr, "/");
|
||||
}
|
||||
}
|
||||
|
@ -1,336 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.util;
|
||||
|
||||
import org.apache.commons.configuration2.Configuration;
|
||||
import org.apache.commons.configuration2.FileBasedConfiguration;
|
||||
import org.apache.commons.configuration2.PropertiesConfiguration;
|
||||
import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
|
||||
import org.apache.commons.configuration2.builder.fluent.Parameters;
|
||||
import org.apache.commons.configuration2.ex.ConfigurationException;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
public class PulsarNBClientConf {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class);
|
||||
|
||||
private String canonicalFilePath = "";
|
||||
|
||||
public static final String SCHEMA_CONF_PREFIX = "schema";
|
||||
public static final String CLIENT_CONF_PREFIX = "client";
|
||||
public static final String PRODUCER_CONF_PREFIX = "producer";
|
||||
public static final String CONSUMER_CONF_PREFIX = "consumer";
|
||||
public static final String READER_CONF_PREFIX = "reader";
|
||||
private final HashMap<String, Object> schemaConfMap = new HashMap<>();
|
||||
private final HashMap<String, Object> clientConfMap = new HashMap<>();
|
||||
private final HashMap<String, Object> producerConfMap = new HashMap<>();
|
||||
private final HashMap<String, Object> consumerConfMap = new HashMap<>();
|
||||
private final HashMap<String, Object> readerConfMap = new HashMap<>();
|
||||
// TODO: add support for other operation types: websocket-producer, managed-ledger
|
||||
|
||||
public PulsarNBClientConf(String fileName) {
|
||||
File file = new File(fileName);
|
||||
|
||||
try {
|
||||
canonicalFilePath = file.getCanonicalPath();
|
||||
|
||||
Parameters params = new Parameters();
|
||||
|
||||
FileBasedConfigurationBuilder<FileBasedConfiguration> builder =
|
||||
new FileBasedConfigurationBuilder<FileBasedConfiguration>(PropertiesConfiguration.class)
|
||||
.configure(params.properties()
|
||||
.setFileName(fileName));
|
||||
|
||||
Configuration config = builder.getConfiguration();
|
||||
|
||||
// Get schema specific configuration settings
|
||||
for (Iterator<String> it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) {
|
||||
String confKey = it.next();
|
||||
String confVal = config.getProperty(confKey).toString();
|
||||
if (!StringUtils.isBlank(confVal))
|
||||
schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey));
|
||||
}
|
||||
|
||||
// Get client connection specific configuration settings
|
||||
for (Iterator<String> it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) {
|
||||
String confKey = it.next();
|
||||
String confVal = config.getProperty(confKey).toString();
|
||||
if (!StringUtils.isBlank(confVal))
|
||||
clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey));
|
||||
}
|
||||
|
||||
// Get producer specific configuration settings
|
||||
for (Iterator<String> it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) {
|
||||
String confKey = it.next();
|
||||
String confVal = config.getProperty(confKey).toString();
|
||||
if (!StringUtils.isBlank(confVal))
|
||||
producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
|
||||
}
|
||||
|
||||
// Get consumer specific configuration settings
|
||||
for (Iterator<String> it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) {
|
||||
String confKey = it.next();
|
||||
String confVal = config.getProperty(confKey).toString();
|
||||
if (!StringUtils.isBlank(confVal))
|
||||
consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
|
||||
}
|
||||
|
||||
// Get reader specific configuration settings
|
||||
for (Iterator<String> it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) {
|
||||
String confKey = it.next();
|
||||
String confVal = config.getProperty(confKey).toString();
|
||||
if (!StringUtils.isBlank(confVal))
|
||||
readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey));
|
||||
}
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Can't read the specified config properties file!");
|
||||
ioe.printStackTrace();
|
||||
} catch (ConfigurationException cex) {
|
||||
logger.error("Error loading configuration items from the specified config properties file: " + canonicalFilePath);
|
||||
cex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//////////////////
|
||||
// Get Schema related config
|
||||
public Map<String, Object> getSchemaConfMap() {
|
||||
return this.schemaConfMap;
|
||||
}
|
||||
public boolean hasSchemaConfKey(String key) {
|
||||
if (key.contains(SCHEMA_CONF_PREFIX))
|
||||
return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return schemaConfMap.containsKey(key);
|
||||
}
|
||||
public Object getSchemaConfValue(String key) {
|
||||
if (key.contains(SCHEMA_CONF_PREFIX))
|
||||
return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1));
|
||||
else
|
||||
return schemaConfMap.get(key);
|
||||
}
|
||||
public void setSchemaConfValue(String key, Object value) {
|
||||
if (key.contains(SCHEMA_CONF_PREFIX))
|
||||
schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value);
|
||||
else
|
||||
schemaConfMap.put(key, value);
|
||||
}
|
||||
|
||||
|
||||
//////////////////
|
||||
// Get Pulsar client related config
|
||||
public Map<String, Object> getClientConfMap() {
|
||||
return this.clientConfMap;
|
||||
}
|
||||
public boolean hasClientConfKey(String key) {
|
||||
if (key.contains(CLIENT_CONF_PREFIX))
|
||||
return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return clientConfMap.containsKey(key);
|
||||
}
|
||||
public Object getClientConfValue(String key) {
|
||||
if (key.contains(CLIENT_CONF_PREFIX))
|
||||
return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1));
|
||||
else
|
||||
return clientConfMap.get(key);
|
||||
}
|
||||
public void setClientConfValue(String key, Object value) {
|
||||
if (key.contains(CLIENT_CONF_PREFIX))
|
||||
clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value);
|
||||
else
|
||||
clientConfMap.put(key, value);
|
||||
}
|
||||
|
||||
|
||||
//////////////////
|
||||
// Get Pulsar producer related config
|
||||
public Map<String, Object> getProducerConfMap() {
|
||||
return this.producerConfMap;
|
||||
}
|
||||
public boolean hasProducerConfKey(String key) {
|
||||
if (key.contains(PRODUCER_CONF_PREFIX))
|
||||
return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return producerConfMap.containsKey(key);
|
||||
}
|
||||
public Object getProducerConfValue(String key) {
|
||||
if (key.contains(PRODUCER_CONF_PREFIX))
|
||||
return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1));
|
||||
else
|
||||
return producerConfMap.get(key);
|
||||
}
|
||||
public void setProducerConfValue(String key, Object value) {
|
||||
if (key.contains(PRODUCER_CONF_PREFIX))
|
||||
producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value);
|
||||
else
|
||||
producerConfMap.put(key, value);
|
||||
}
|
||||
// other producer helper functions ...
|
||||
public String getProducerName() {
|
||||
Object confValue = getProducerConfValue(
|
||||
"producer." + PulsarActivityUtil.PRODUCER_CONF_STD_KEY.producerName.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
public String getProducerTopicName() {
|
||||
Object confValue = getProducerConfValue(
|
||||
"producer." + PulsarActivityUtil.PRODUCER_CONF_STD_KEY.topicName);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
|
||||
|
||||
//////////////////
|
||||
// Get Pulsar consumer related config
|
||||
public Map<String, Object> getConsumerConfMap() {
|
||||
return this.consumerConfMap;
|
||||
}
|
||||
public boolean hasConsumerConfKey(String key) {
|
||||
if (key.contains(CONSUMER_CONF_PREFIX))
|
||||
return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return consumerConfMap.containsKey(key);
|
||||
}
|
||||
public Object getConsumerConfValue(String key) {
|
||||
if (key.contains(CONSUMER_CONF_PREFIX))
|
||||
return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return consumerConfMap.get(key);
|
||||
}
|
||||
public void setConsumerConfValue(String key, Object value) {
|
||||
if (key.contains(CONSUMER_CONF_PREFIX))
|
||||
consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value);
|
||||
else
|
||||
consumerConfMap.put(key, value);
|
||||
}
|
||||
// Other consumer helper functions ...
|
||||
public String getConsumerTopicNames() {
|
||||
Object confValue = getConsumerConfValue(
|
||||
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
public String getConsumerTopicPattern() {
|
||||
Object confValue = getConsumerConfValue(
|
||||
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
public String getConsumerSubscriptionName() {
|
||||
Object confValue = getConsumerConfValue(
|
||||
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
public String getConsumerSubscriptionType() {
|
||||
Object confValue = getConsumerConfValue(
|
||||
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
public String getConsumerName() {
|
||||
Object confValue = getConsumerConfValue(
|
||||
"consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
// NOTE: Below are not a standard Pulsar consumer configuration parameter as
|
||||
// listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer"
|
||||
// They're custom-made configuration properties for NB pulsar driver consumer.
|
||||
public int getConsumerTimeoutSeconds() {
|
||||
Object confValue = getConsumerConfValue(
|
||||
"consumer." + PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label);
|
||||
if (confValue == null)
|
||||
return -1; // infinite
|
||||
else
|
||||
return Integer.parseInt(confValue.toString());
|
||||
}
|
||||
|
||||
//////////////////
|
||||
// Get Pulsar reader related config
|
||||
public Map<String, Object> getReaderConfMap() {
|
||||
return this.readerConfMap;
|
||||
}
|
||||
public boolean hasReaderConfKey(String key) {
|
||||
if (key.contains(READER_CONF_PREFIX))
|
||||
return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return readerConfMap.containsKey(key);
|
||||
}
|
||||
public Object getReaderConfValue(String key) {
|
||||
if (key.contains(READER_CONF_PREFIX))
|
||||
return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1));
|
||||
else
|
||||
return readerConfMap.get(key);
|
||||
}
|
||||
public void setReaderConfValue(String key, Object value) {
|
||||
if (key.contains(READER_CONF_PREFIX))
|
||||
readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value);
|
||||
else
|
||||
readerConfMap.put(key, value);
|
||||
}
|
||||
// Other reader helper functions ...
|
||||
public String getReaderTopicName() {
|
||||
Object confValue = getReaderConfValue(
|
||||
"reader." + PulsarActivityUtil.READER_CONF_STD_KEY.topicName.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
public String getReaderName() {
|
||||
Object confValue = getReaderConfValue(
|
||||
"reader." + PulsarActivityUtil.READER_CONF_STD_KEY.readerName.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
// NOTE: Below are not a standard Pulsar reader configuration parameter as
|
||||
// listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader"
|
||||
// They're custom-made configuration properties for NB pulsar driver reader.
|
||||
public String getStartMsgPosStr() {
|
||||
Object confValue = getReaderConfValue(
|
||||
"reader." + PulsarActivityUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label);
|
||||
if (confValue == null)
|
||||
return "";
|
||||
else
|
||||
return confValue.toString();
|
||||
}
|
||||
}
|
@ -1,36 +0,0 @@
|
||||
#
|
||||
# Results:
|
||||
# - 10 tenants
|
||||
# - 2 namespaces per tanant
|
||||
# - 5 topics per namespace
|
||||
#------------------------------------------------------
|
||||
#tenant=tenant_0 namespace=default_0 core_topic_name=t0
|
||||
#tenant=tenant_0 namespace=default_0 core_topic_name=t1
|
||||
#tenant=tenant_0 namespace=default_0 core_topic_name=t2
|
||||
#tenant=tenant_0 namespace=default_0 core_topic_name=t3
|
||||
#tenant=tenant_0 namespace=default_0 core_topic_name=t4
|
||||
#tenant=tenant_0 namespace=default_1 core_topic_name=t0
|
||||
#tenant=tenant_0 namespace=default_1 core_topic_name=t1
|
||||
#tenant=tenant_0 namespace=default_1 core_topic_name=t2
|
||||
#tenant=tenant_0 namespace=default_1 core_topic_name=t3
|
||||
#tenant=tenant_0 namespace=default_1 core_topic_name=t4
|
||||
#tenant=tenant_1 namespace=default_0 core_topic_name=t0
|
||||
#tenant=tenant_1 namespace=default_0 core_topic_name=t1
|
||||
#tenant=tenant_1 namespace=default_0 core_topic_name=t2
|
||||
#tenant=tenant_1 namespace=default_0 core_topic_name=t3
|
||||
#tenant=tenant_1 namespace=default_0 core_topic_name=t4
|
||||
#tenant=tenant_1 namespace=default_1 core_topic_name=t0
|
||||
#tenant=tenant_1 namespace=default_1 core_topic_name=t1
|
||||
#tenant=tenant_1 namespace=default_1 core_topic_name=t2
|
||||
#tenant=tenant_1 namespace=default_1 core_topic_name=t3
|
||||
#tenant=tenant_1 namespace=default_1 core_topic_name=t4
|
||||
# ... ...
|
||||
|
||||
bindings:
|
||||
# message key and value
|
||||
#mykey: NumberNameToString()
|
||||
#myvalue: AlphaNumericString(20)
|
||||
# Admin API - create tenant, namespace, and topic
|
||||
tenant: Mod(100); Div(10L); ToString(); Prefix("tenant_")
|
||||
namespace: Mod(10); Div(5L); ToString(); Prefix("default_")
|
||||
core_topic_name: Mod(5); ToString(); Prefix("t")
|
@ -1,52 +0,0 @@
|
||||
### Schema related configurations - schema.xxx
|
||||
# valid types:
|
||||
# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type)
|
||||
# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue)
|
||||
# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct)
|
||||
# avro, json, protobuf
|
||||
#
|
||||
# TODO: as a starting point, only supports the following types
|
||||
# 1) primitive types, including bytearray (byte[]) which is default, for messages without schema
|
||||
# 2) Avro for messages with schema
|
||||
#schema.type=avro
|
||||
#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc
|
||||
schema.type=
|
||||
schema.definition=
|
||||
|
||||
|
||||
### Pulsar client related configurations - client.xxx
|
||||
# http://pulsar.apache.org/docs/en/client-libraries-java/#client
|
||||
client.connectionTimeoutMs=5000
|
||||
client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken
|
||||
# Cluster admin
|
||||
client.authParams=
|
||||
client.tlsAllowInsecureConnection=true
|
||||
|
||||
|
||||
### Producer related configurations (global) - producer.xxx
|
||||
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer
|
||||
producer.producerName=
|
||||
producer.topicName=
|
||||
producer.sendTimeoutMs=
|
||||
producer.blockIfQueueFull=true
|
||||
producer.maxPendingMessages=5000
|
||||
producer.batchingMaxMessages=5000
|
||||
|
||||
|
||||
### Consumer related configurations (global) - consumer.xxx
|
||||
# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer
|
||||
consumer.topicNames=
|
||||
consumer.topicsPattern=
|
||||
consumer.subscriptionName=
|
||||
consumer.subscriptionType=
|
||||
consumer.consumerName=
|
||||
consumer.receiverQueueSize=
|
||||
|
||||
|
||||
### Reader related configurations (global) - reader.xxx
|
||||
# https://pulsar.apache.org/docs/en/client-libraries-java/#reader
|
||||
# - valid Pos: earliest, latest, custom::file://<path>/<to>/<message_id_file>
|
||||
reader.topicName=
|
||||
reader.receiverQueueSize=
|
||||
reader.readerName=
|
||||
reader.startMessagePos=earliest
|
@ -1,11 +0,0 @@
|
||||
{
|
||||
"type": "record",
|
||||
"name": "IotSensor",
|
||||
"namespace": "TestNS",
|
||||
"fields" : [
|
||||
{"name": "SensorID", "type": "string"},
|
||||
{"name": "SensorType", "type": "string"},
|
||||
{"name": "ReadingTime", "type": "string"},
|
||||
{"name": "ReadingValue", "type": "float"}
|
||||
]
|
||||
}
|
@ -1,22 +0,0 @@
|
||||
bindings:
|
||||
# 20 namespaces: 10 tenants, 2 namespaces/tenant
|
||||
tenant: Mod(20); Div(2L); ToString(); Prefix("tnt")
|
||||
namespace: Mod(2); ToString(); Prefix("ns")
|
||||
|
||||
params:
|
||||
# "true" - asynchronous Pulsar Admin API
|
||||
# "false" - synchronous Pulsar Admin API
|
||||
async_api: "false"
|
||||
# "true" - delete namespace
|
||||
# "false" - create namespace
|
||||
admin_delop: "false"
|
||||
|
||||
blocks:
|
||||
- name: create-namespace-block
|
||||
tags:
|
||||
phase: admin-namespace
|
||||
admin_task: true
|
||||
statements:
|
||||
- name: s1
|
||||
optype: admin-namespace
|
||||
namespace: "{tenant}/{namespace}"
|
@ -1,23 +0,0 @@
|
||||
bindings:
|
||||
# 10 tenants
|
||||
tenant: Mod(10); ToString(); Prefix("tnt")
|
||||
|
||||
params:
|
||||
# "true" - asynchronous Pulsar Admin API
|
||||
# "false" - synchronous Pulsar Admin API
|
||||
async_api: "false"
|
||||
# "true" - delete tenant
|
||||
# "false" - create tenant
|
||||
admin_delop: "true"
|
||||
|
||||
blocks:
|
||||
- name: create-tenant-block
|
||||
tags:
|
||||
phase: admin-tenant
|
||||
admin_task: true
|
||||
statements:
|
||||
- name: s1
|
||||
optype: admin-tenant
|
||||
# admin_roles:
|
||||
# allowed_clusters:
|
||||
tenant: "{tenant}"
|
@ -1,25 +0,0 @@
|
||||
bindings:
|
||||
# 100 topics: 10 tenants, 2 namespaces/tenant, 5 topics/namespace
|
||||
tenant: Mod(100); Div(10L); ToString(); Prefix("tnt")
|
||||
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
|
||||
core_topic_name: Mod(5); ToString(); Prefix("t")
|
||||
|
||||
params:
|
||||
topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
|
||||
# "true" - asynchronous Pulsar Admin API
|
||||
# "false" - synchronous Pulsar Admin API
|
||||
async_api: "false"
|
||||
# "true" - delete topic
|
||||
# "false" - create topic
|
||||
admin_delop: "false"
|
||||
|
||||
blocks:
|
||||
- name: create-topic-block
|
||||
tags:
|
||||
phase: admin-topic
|
||||
admin_task: true
|
||||
statements:
|
||||
- name: s1
|
||||
optype: admin-topic
|
||||
enable_partition: "false"
|
||||
partition_num: "5"
|
@ -1,93 +0,0 @@
|
||||
bindings:
|
||||
# message key and value
|
||||
mykey: NumberNameToString()
|
||||
sensor_id: ToUUID();ToString();
|
||||
reading_time: ToDateTime();
|
||||
reading_value: ToFloat(100);
|
||||
tenant: Mod(100); Div(10L); ToString(); Prefix("tnt")
|
||||
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
|
||||
core_topic_name: Mod(5); ToString(); Prefix("t")
|
||||
|
||||
# document level parameters that apply to all Pulsar client types:
|
||||
params:
|
||||
topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
|
||||
# topic_uri: "persistent://public/default/mytopic"
|
||||
async_api: "true"
|
||||
|
||||
blocks:
|
||||
- name: batch-producer-block
|
||||
tags:
|
||||
phase: batch-producer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: batch-msg-send-start
|
||||
# For batch producer, "producer_name" should be associated with batch start
|
||||
# batch_producer_name: {batch_producer_name}
|
||||
ratio: 1
|
||||
- name: s2
|
||||
optype: batch-msg-send
|
||||
msg_key: "{mykey}"
|
||||
msg_value: |
|
||||
{
|
||||
"SensorID": "{sensor_id}",
|
||||
"SensorType": "Temperature",
|
||||
"ReadingTime": "{reading_time}",
|
||||
"ReadingValue": {reading_value}
|
||||
}
|
||||
ratio: 100
|
||||
- name: s3
|
||||
optype: batch-msg-send-end
|
||||
ratio: 1
|
||||
|
||||
- name: producer-block
|
||||
tags:
|
||||
phase: producer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-send
|
||||
# producer_name: {producer_name}
|
||||
msg_key: "{mykey}"
|
||||
msg_property: "{myprop}"
|
||||
msg_value: |
|
||||
{
|
||||
"SensorID": "{sensor_id}",
|
||||
"SensorType": "Temperature",
|
||||
"ReadingTime": "{reading_time}",
|
||||
"ReadingValue": {reading_value}
|
||||
}
|
||||
|
||||
- name: consumer-block
|
||||
tags:
|
||||
phase: consumer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-consume
|
||||
# topic_names:
|
||||
# topics_pattern:
|
||||
subscription_name: "mysub"
|
||||
# subscription_type:
|
||||
# consumer_name:
|
||||
|
||||
- name: reader-block
|
||||
tags:
|
||||
phase: reader
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-read
|
||||
# reader_name:
|
||||
|
||||
# - websocket-producer:
|
||||
# tags:
|
||||
# type: websocket-produer
|
||||
# statements:
|
||||
# - websocket-producer-stuff:
|
||||
#
|
||||
# - managed-ledger:
|
||||
# tags:
|
||||
# type: managed-ledger
|
||||
# statement:
|
||||
# - managed-ledger-stuff:
|
@ -1,96 +0,0 @@
|
||||
bindings:
|
||||
# message key, property and value
|
||||
mykey: NumberNameToString()
|
||||
int_prop_val: ToString(); Prefix("IntProp_")
|
||||
text_prop_val: AlphaNumericString(10); Prefix("TextProp_")
|
||||
myvalue: NumberNameToString() #AlphaNumericString(20)
|
||||
# tenant, namespace, and core topic name (without tenant and namespace)
|
||||
tenant: Mod(100); Div(10L); ToString(); Prefix("tnt")
|
||||
namespace: Mod(10); Div(5L); ToString(); Prefix("ns")
|
||||
core_topic_name: Mod(5); ToString(); Prefix("t")
|
||||
|
||||
# document level parameters that apply to all Pulsar client types:
|
||||
params:
|
||||
topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}"
|
||||
async_api: "true"
|
||||
|
||||
blocks:
|
||||
- name: batch-producer-block
|
||||
tags:
|
||||
phase: batch-producer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: batch-msg-send-start
|
||||
# For batch producer, "producer_name" should be associated with batch start
|
||||
# batch_producer_name: {batch_producer_name}
|
||||
ratio: 1
|
||||
- name: s2
|
||||
optype: batch-msg-send
|
||||
msg_key: "{mykey}"
|
||||
msg_property: |
|
||||
{
|
||||
"prop1": "{int_prop_val}",
|
||||
"prop2": "{text_prop_val}}"
|
||||
}
|
||||
msg_value: "{myvalue}"
|
||||
ratio: 100
|
||||
- name: s3
|
||||
optype: batch-msg-send-end
|
||||
ratio: 1
|
||||
|
||||
- name: producer-block
|
||||
tags:
|
||||
phase: producer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-send
|
||||
# producer_name: {producer_name}
|
||||
# msg_key:
|
||||
msg_value: "{myvalue}"
|
||||
|
||||
- name: consumer-block
|
||||
tags:
|
||||
phase: consumer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-consume
|
||||
subscription_name: "mysub"
|
||||
# subscription_type:
|
||||
# consumer_name:
|
||||
|
||||
- name: reader-block
|
||||
tags:
|
||||
phase: reader
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-read
|
||||
# reader_name:
|
||||
|
||||
- name: multi-topic-consumer-block
|
||||
tags:
|
||||
phase: multi-topic-consumer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-mt-consume
|
||||
# topic_names:
|
||||
# topics_pattern:
|
||||
subscription_name: "mysub"
|
||||
# subscription_type:
|
||||
# consumer_name:
|
||||
|
||||
# - websocket-producer:
|
||||
# tags:
|
||||
# type: websocket-producer
|
||||
# statements:
|
||||
# - websocket-producer-stuff:
|
||||
#
|
||||
# - managed-ledger:
|
||||
# tags:
|
||||
# type: managed-ledger
|
||||
# statement:
|
||||
# - managed-ledger-stuff:
|
@ -1,30 +0,0 @@
|
||||
bindings:
|
||||
# message key, property and value
|
||||
myprop1: AlphaNumericString(10); Prefix("PropVal_")
|
||||
myvalue: NumberNameToString() #AlphaNumericString(20)
|
||||
|
||||
# document level parameters that apply to all Pulsar client types:
|
||||
params:
|
||||
topic_uri: "persistent://public/default/sanity_e2e_2"
|
||||
async_api: "true"
|
||||
|
||||
blocks:
|
||||
- name: e2e-msg-proc-block
|
||||
tags:
|
||||
phase: e2e-msg-proc
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: ec2-msg-proc-send
|
||||
# msg_key:
|
||||
msg_property: |
|
||||
{
|
||||
"prop1": "{myprop1}"
|
||||
}
|
||||
msg_value: "{myvalue}"
|
||||
ratio: 1
|
||||
- name: s2
|
||||
optype: ec2-msg-proc-consume
|
||||
ratio: 1
|
||||
subscription_name: "mysub"
|
||||
# subscription_type:
|
@ -1,39 +0,0 @@
|
||||
bindings:
|
||||
# message key, property and value
|
||||
myprop1: AlphaNumericString(10)
|
||||
myvalue: NumberNameToString()
|
||||
|
||||
# document level parameters that apply to all Pulsar client types:
|
||||
params:
|
||||
topic_uri: "persistent://tnt0/ns0/sanity_seqloss12"
|
||||
# Only applicable to producer and consumer
|
||||
# - used for message ordering and message loss check
|
||||
async_api: "true"
|
||||
seq_tracking: "true"
|
||||
|
||||
blocks:
|
||||
- name: producer-block
|
||||
tags:
|
||||
phase: producer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-send
|
||||
#seqerr_simu: "out_of_order"
|
||||
#seqerr_simu: "msg_loss"
|
||||
#seqerr_simu: "msg_dup"
|
||||
#seqerr_simu: "out_of_order, msg_loss"
|
||||
# msg_key:
|
||||
# msg_property:
|
||||
msg_value: "{myvalue}"
|
||||
|
||||
- name: consumer-block
|
||||
tags:
|
||||
phase: consumer
|
||||
admin_task: false
|
||||
statements:
|
||||
- name: s1
|
||||
optype: msg-consume
|
||||
subscription_name: "mysub"
|
||||
subscription_type: "Shared"
|
||||
# consumer_name:
|
@ -1,89 +0,0 @@
|
||||
# TODO : Design Revisit -- Advanced Driver Features
|
||||
|
||||
**NOTE**: The following text is based on the original multi-layer API
|
||||
caching design which is not fully implemented at the moment. We need to
|
||||
revisit the original design at some point in order to achieve maximum
|
||||
testing flexibility.
|
||||
|
||||
To summarize, the original caching design has the following key
|
||||
requirements:
|
||||
|
||||
* **Requirement 1**: Each NB Pulsar activity is able to launch and cache
|
||||
multiple **client spaces**
|
||||
* **Requirement 2**: Each client space can launch and cache multiple
|
||||
Pulsar operators of the same type (producer, consumer, etc.)
|
||||
* **Requirement 3**: The size of each Pulsar operator specific cached
|
||||
space can be configurable.
|
||||
|
||||
In the current implementation, only requirement 2 is implemented.
|
||||
|
||||
* For requirement 1, the current implementation only supports one client
|
||||
space per NB Pulsar activity
|
||||
* For requirement 3, the cache space size is not configurable (no limit at
|
||||
the moment)
|
||||
|
||||
## Other Activity Parameters
|
||||
|
||||
- **maxcached** - A default value to be applied to `max_clients`,
|
||||
`max_producers`, `max_consumers`.
|
||||
- default: `max_cached=100`
|
||||
- **max_clients** - Clients cache size. This is the number of client
|
||||
instances which are allowed to be cached in the NoSQLBench client
|
||||
runtime. The clients cache automatically maintains a cache of unique
|
||||
client instances internally. default: _maxcached_
|
||||
- **max_operators** - Producers/Consumers/Readers cache size (per client
|
||||
instance). Limits the number of instances which are allowed to be cached
|
||||
per client instance. default: _maxcached_
|
||||
|
||||
## API Caching
|
||||
|
||||
This driver is tailored around the multi-tenancy and topic naming scheme
|
||||
that is part of Apache Pulsar. Specifically, you can create an arbitrary
|
||||
number of client instances, producers (per client), and consumers (per
|
||||
client) depending on your testing requirements.
|
||||
|
||||
Further, the topic URI is composed from the provided qualifiers of
|
||||
`persistence`, `tenant`, `namespace`, and `topic`, or you can provide a
|
||||
fully-composed value in the `persistence://tenant/namespace/topic`
|
||||
form.
|
||||
|
||||
### Instancing Controls
|
||||
|
||||
Normative usage of the Apache Pulsar API follows a strictly enforced
|
||||
binding of topics to producers and consumers. As well, clients may be
|
||||
customized with different behavior for advanced testing scenarios. There
|
||||
is a significant variety of messaging and concurrency schemes seen in
|
||||
modern architectures. Thus, it is important that testing tools rise to the
|
||||
occasion by letting users configure their testing runtimes to emulate
|
||||
applications as they are found in practice. To this end, the NoSQLBench
|
||||
driver for Apache Pulsar provides a set controls within its op template
|
||||
format which allow for flexible yet intuitive instancing in the client
|
||||
runtime. This is enabled directly by using nominative variables for
|
||||
instance names where needed. When the instance names are not provided for
|
||||
an operation, defaults are used to emulate a simple configuration.
|
||||
|
||||
Since this is a new capability in a NoSQLBench driver, how it works is
|
||||
explained below:
|
||||
|
||||
When a pulsar cycles is executed, the operation is synthesized from the op
|
||||
template fields as explained below under _Op Fields_. This happens in a
|
||||
specific order:
|
||||
|
||||
1. The client instance name is resolved. If a `client` field is provided,
|
||||
this is taken as the client instance name. If not, it is set
|
||||
to `default`.
|
||||
2. The named client instance is fetched from the cache, or created and
|
||||
cached if it does not yet exist.
|
||||
3. The topic_uri is resolved. This is the value to be used with
|
||||
`.topic(...)` calls in the API. The op fields below explain how to
|
||||
control this value.
|
||||
4. For _send_ operations, a producer is named and created if needed. By
|
||||
default, the producer is named after the topic_uri above. You can
|
||||
override this by providing a value for `producer`.
|
||||
5. For _recv_ operations, a consumer is named and created if needed. By
|
||||
default, the consumer is named after the topic_uri above. You can
|
||||
override this by providing a value for `consumer`.
|
||||
|
||||
The most important detail for understanding the instancing controls is
|
||||
that clients, producers, and consumers are all named and cached in the
|
||||
specific order above.
|
@ -1,34 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class TestYamlLoader {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(TestYamlLoader.class);
|
||||
|
||||
@Test
|
||||
public void loadAvroYaml() {
|
||||
RawStmtsLoader sl = new RawStmtsLoader();
|
||||
RawStmtsDocList rsdl = sl.loadPath(logger, "activities/pulsar_client_avro.yaml");
|
||||
}
|
||||
}
|
@ -1,90 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class MessageSequenceNumberSendingHandlerTest {
|
||||
MessageSequenceNumberSendingHandler sequenceNumberSendingHandler = new MessageSequenceNumberSendingHandler();
|
||||
|
||||
@Test
|
||||
void shouldAddMonotonicSequence() {
|
||||
for (long l = 1; l <= 100; l++) {
|
||||
assertEquals(l, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldInjectMessageLoss() {
|
||||
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
assertEquals(3L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.MsgLoss), 100));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldInjectMessageDuplication() {
|
||||
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.MsgDup), 100));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldInjectMessageOutOfOrder() {
|
||||
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
assertEquals(4L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.OutOfOrder), 100));
|
||||
assertEquals(2L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
assertEquals(3L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
assertEquals(5L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
assertEquals(6, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldInjectOneOfTheSimulatedErrorsRandomly() {
|
||||
Set<PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE> allErrorTypes = new HashSet<>(Arrays.asList(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.values()));
|
||||
|
||||
assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet()));
|
||||
long previousSequenceNumber = 1L;
|
||||
int outOfSequenceInjectionCounter = 0;
|
||||
int messageDupCounter = 0;
|
||||
int messageLossCounter = 0;
|
||||
int successCounter = 0;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
long nextSequenceNumber = sequenceNumberSendingHandler.getNextSequenceNumber(allErrorTypes);
|
||||
if (nextSequenceNumber >= previousSequenceNumber + 3) {
|
||||
outOfSequenceInjectionCounter++;
|
||||
} else if (nextSequenceNumber <= previousSequenceNumber) {
|
||||
messageDupCounter++;
|
||||
} else if (nextSequenceNumber >= previousSequenceNumber + 2) {
|
||||
messageLossCounter++;
|
||||
} else if (nextSequenceNumber == previousSequenceNumber + 1) {
|
||||
successCounter++;
|
||||
}
|
||||
previousSequenceNumber = nextSequenceNumber;
|
||||
}
|
||||
assertTrue(outOfSequenceInjectionCounter > 0);
|
||||
assertTrue(messageDupCounter > 0);
|
||||
assertTrue(messageLossCounter > 0);
|
||||
assertEquals(1000, outOfSequenceInjectionCounter + messageDupCounter + messageLossCounter + successCounter);
|
||||
}
|
||||
|
||||
}
|
@ -1,246 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.driver.pulsar.ops;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
|
||||
class ReceivedMessageSequenceTrackerTest {
|
||||
Counter msgErrOutOfSeqCounter = new Counter();
|
||||
Counter msgErrDuplicateCounter = new Counter();
|
||||
Counter msgErrLossCounter = new Counter();
|
||||
ReceivedMessageSequenceTracker messageSequenceTracker = new ReceivedMessageSequenceTracker(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter, 20, 20);
|
||||
|
||||
@Test
|
||||
void shouldCountersBeZeroWhenSequenceDoesntContainGaps() {
|
||||
// when
|
||||
for (long l = 0; l < 100L; l++) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
// then
|
||||
assertEquals(0, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(0, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L})
|
||||
void shouldDetectMsgLossWhenEverySecondMessageIsLost(long totalMessages) {
|
||||
doShouldDetectMsgLoss(totalMessages, 2);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L})
|
||||
void shouldDetectMsgLossWhenEveryThirdMessageIsLost(long totalMessages) {
|
||||
doShouldDetectMsgLoss(totalMessages, 3);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(longs = {20L, 21L, 40L, 41L, 42L, 43L, 100L})
|
||||
void shouldDetectMsgLossWhenEvery21stMessageIsLost(long totalMessages) {
|
||||
doShouldDetectMsgLoss(totalMessages, 21);
|
||||
}
|
||||
|
||||
private void doShouldDetectMsgLoss(long totalMessages, int looseEveryNthMessage) {
|
||||
int messagesLost = 0;
|
||||
// when
|
||||
boolean lastMessageWasLost = false;
|
||||
for (long l = 0; l < totalMessages; l++) {
|
||||
if (l % looseEveryNthMessage == 1) {
|
||||
messagesLost++;
|
||||
lastMessageWasLost = true;
|
||||
continue;
|
||||
} else {
|
||||
lastMessageWasLost = false;
|
||||
}
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
if (lastMessageWasLost) {
|
||||
messageSequenceTracker.sequenceNumberReceived(totalMessages);
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
// then
|
||||
assertEquals(0, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(messagesLost, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L})
|
||||
void shouldDetectMsgDuplication(long totalMessages) {
|
||||
int messagesDuplicated = 0;
|
||||
// when
|
||||
for (long l = 0; l < totalMessages; l++) {
|
||||
if (l % 2 == 1) {
|
||||
messagesDuplicated++;
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
if (totalMessages % 2 == 0) {
|
||||
messageSequenceTracker.sequenceNumberReceived(totalMessages);
|
||||
}
|
||||
if (totalMessages < 2 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers()) {
|
||||
messageSequenceTracker.close();
|
||||
}
|
||||
|
||||
// then
|
||||
assertEquals(0, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(messagesDuplicated, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(0, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDetectSingleMessageOutOfSequence() {
|
||||
// when
|
||||
for (long l = 0; l < 10L; l++) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
messageSequenceTracker.sequenceNumberReceived(10L);
|
||||
messageSequenceTracker.sequenceNumberReceived(12L);
|
||||
messageSequenceTracker.sequenceNumberReceived(11L);
|
||||
for (long l = 13L; l < 100L; l++) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
|
||||
// then
|
||||
assertEquals(1, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(0, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDetectMultipleMessagesOutOfSequence() {
|
||||
// when
|
||||
for (long l = 0; l < 10L; l++) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
messageSequenceTracker.sequenceNumberReceived(10L);
|
||||
messageSequenceTracker.sequenceNumberReceived(14L);
|
||||
messageSequenceTracker.sequenceNumberReceived(13L);
|
||||
messageSequenceTracker.sequenceNumberReceived(11L);
|
||||
messageSequenceTracker.sequenceNumberReceived(12L);
|
||||
for (long l = 15L; l < 100L; l++) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
|
||||
// then
|
||||
assertEquals(2, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(0, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDetectIndividualMessageLoss() {
|
||||
// when
|
||||
for (long l = 0; l < 100L; l++) {
|
||||
if (l != 11L) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
|
||||
// then
|
||||
assertEquals(0, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(1, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDetectGapAndMessageDuplication() {
|
||||
// when
|
||||
for (long l = 0; l < 100L; l++) {
|
||||
if (l != 11L) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
if (l == 12L) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
|
||||
// then
|
||||
assertEquals(0, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(1, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(1, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDetectGapAndMessageDuplicationTimes2() {
|
||||
// when
|
||||
for (long l = 0; l < 100L; l++) {
|
||||
if (l != 11L) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
if (l == 12L) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
|
||||
// then
|
||||
assertEquals(0, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(2, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(1, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void shouldDetectDelayedOutOfOrderDelivery() {
|
||||
// when
|
||||
for (long l = 0; l < 5 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers(); l++) {
|
||||
if (l != 10) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
if (l == messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers() * 2) {
|
||||
messageSequenceTracker.sequenceNumberReceived(10);
|
||||
}
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
|
||||
// then
|
||||
assertEquals(1, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(0, msgErrLossCounter.getCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldDetectDelayedOutOfOrderDeliveryOf2ConsecutiveSequenceNumbers() {
|
||||
// when
|
||||
for (long l = 0; l < 5 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers(); l++) {
|
||||
if (l != 10 && l != 11) {
|
||||
messageSequenceTracker.sequenceNumberReceived(l);
|
||||
}
|
||||
if (l == messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers() * 2) {
|
||||
messageSequenceTracker.sequenceNumberReceived(10);
|
||||
messageSequenceTracker.sequenceNumberReceived(11);
|
||||
}
|
||||
}
|
||||
messageSequenceTracker.close();
|
||||
|
||||
// then
|
||||
assertEquals(2, msgErrOutOfSeqCounter.getCount());
|
||||
assertEquals(0, msgErrDuplicateCounter.getCount());
|
||||
assertEquals(0, msgErrLossCounter.getCount());
|
||||
}
|
||||
}
|
@ -22,9 +22,11 @@ public class BundledMarkdownLoader {
|
||||
|
||||
public static DocsBinder loadBundledMarkdown() {
|
||||
ServiceLoader<BundledMarkdownManifest> loader = ServiceLoader.load(BundledMarkdownManifest.class);
|
||||
Docs docs = new Docs();
|
||||
DocsBinder docs = new Docs();
|
||||
for (BundledMarkdownManifest docPathInfos : loader) {
|
||||
docs.merge(docPathInfos.getDocs());
|
||||
|
||||
DocsBinder docsBinder = docPathInfos.getDocs();
|
||||
docs = docs.merge(docsBinder);
|
||||
}
|
||||
|
||||
return docs;
|
||||
|
@ -51,6 +51,11 @@ public class MutableMarkdown {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
public MutableMarkdown(String rawMarkdown) {
|
||||
this.path = null;
|
||||
this.rawMarkdown = rawMarkdown;
|
||||
parseStructure(rawMarkdown);
|
||||
}
|
||||
|
||||
private void parseStructure(String rawMarkdown) {
|
||||
AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor();
|
||||
@ -69,13 +74,19 @@ public class MutableMarkdown {
|
||||
} else if (node instanceof WhiteSpace) {
|
||||
} else if (node instanceof YamlFrontMatterBlock) {
|
||||
} else {
|
||||
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName());
|
||||
if(this.path != null)
|
||||
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName());
|
||||
else
|
||||
throw new RuntimeException("The markdown string provided must have an initial heading as a title, before any other element, but found: "+ node.getClass().getSimpleName());
|
||||
}
|
||||
node=node.getNext();
|
||||
}
|
||||
}
|
||||
if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) {
|
||||
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title.");
|
||||
if(this.path != null)
|
||||
throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title.");
|
||||
else
|
||||
throw new RuntimeException("The markdown string provided has no heading to use as a title.");
|
||||
}
|
||||
}
|
||||
|
||||
@ -90,7 +101,10 @@ public class MutableMarkdown {
|
||||
if (end>=0) {
|
||||
return rawMarkdown.substring(end+4);
|
||||
} else {
|
||||
throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary);
|
||||
if(path != null)
|
||||
throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary);
|
||||
else
|
||||
throw new RuntimeException("Unable to find matching boundaries in provided markdown: " + boundary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -72,6 +72,7 @@
|
||||
<artifactId>adapter-diag</artifactId>
|
||||
<version>4.17.32-SNAPSHOT</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
@ -39,7 +39,7 @@ public class BundledFrontmatterInjector implements BundledMarkdownProcessor {
|
||||
if (name.length()>i) {
|
||||
int ord = name.charAt(i) - 'a';
|
||||
double addend = Math.pow(pow, i) * ord;
|
||||
sum += addend;
|
||||
sum += (int)addend;
|
||||
} else {
|
||||
break;
|
||||
}
|
@ -53,6 +53,7 @@ public class BundledMarkdownExporter implements BundledApp {
|
||||
String zipfile = options.valueOf(zipfileSpec);
|
||||
|
||||
new BundledMarkdownZipExporter(new BundledFrontmatterInjector()).exportDocs(Path.of(zipfile));
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
@ -20,6 +20,7 @@ import io.nosqlbench.api.docsapi.BundledMarkdownLoader;
|
||||
import io.nosqlbench.api.docsapi.DocsBinder;
|
||||
import io.nosqlbench.api.docsapi.DocsNameSpace;
|
||||
import io.nosqlbench.api.markdown.aggregator.MutableMarkdown;
|
||||
import io.nosqlbench.virtdata.userlibs.apps.docsapp.VirtDataGenDocsApp;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@ -29,7 +30,12 @@ import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.Date;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.function.Function;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
@ -51,12 +57,33 @@ public class BundledMarkdownZipExporter {
|
||||
zipstream.setMethod(ZipOutputStream.DEFLATED);
|
||||
zipstream.setLevel(9);
|
||||
|
||||
DocsBinder docsNameSpaces = BundledMarkdownLoader.loadBundledMarkdown();
|
||||
DocsBinder docsNameSpaces = BundledMarkdownLoader.loadBundledMarkdown(); //Loads the drivers under @Service Annotation
|
||||
|
||||
for (DocsNameSpace docs_ns : docsNameSpaces) {
|
||||
for (Path p : docs_ns) {
|
||||
addEntry(p, p.getParent(), zipstream);
|
||||
addEntry(p, p.getParent(), zipstream, docs_ns.getName() + "/");
|
||||
}
|
||||
}
|
||||
|
||||
ExecutorService executorService = Executors.newSingleThreadExecutor();
|
||||
Future<Map<String, StringBuilder>> future = executorService.submit(new VirtDataGenDocsApp(null));
|
||||
Map<String, StringBuilder> builderMap = future.get();
|
||||
executorService.shutdown();
|
||||
String bindingsPrefix ="bindings/";
|
||||
for(Map.Entry<String, StringBuilder> entry : builderMap.entrySet())
|
||||
{
|
||||
String filename = entry.getKey();
|
||||
StringBuilder fileStringBuilder = entry.getValue();
|
||||
MutableMarkdown parsed = new MutableMarkdown(fileStringBuilder.toString());
|
||||
for (BundledMarkdownProcessor filter : this.filters) {
|
||||
parsed = filter.apply(parsed);
|
||||
}
|
||||
ZipEntry zipEntry = new ZipEntry(bindingsPrefix + filename);
|
||||
zipEntry.setTime(new Date().getTime());
|
||||
zipstream.putNextEntry(zipEntry);
|
||||
zipstream.write(parsed.getComposedMarkdown().getBytes(StandardCharsets.UTF_8));
|
||||
zipstream.closeEntry();
|
||||
}
|
||||
zipstream.finish();
|
||||
stream.close();
|
||||
} catch (Exception e) {
|
||||
@ -64,18 +91,17 @@ public class BundledMarkdownZipExporter {
|
||||
}
|
||||
}
|
||||
|
||||
private void addEntry(Path p, Path r, ZipOutputStream zos) throws IOException {
|
||||
private void addEntry(Path p, Path r, ZipOutputStream zos, String prefix) throws IOException {
|
||||
|
||||
String name = r.relativize(p).toString();
|
||||
name = Files.isDirectory(p) ? (name.endsWith(File.separator) ? name : name + File.separator) : name;
|
||||
|
||||
ZipEntry entry = new ZipEntry(name);
|
||||
ZipEntry entry = new ZipEntry(prefix + name);
|
||||
|
||||
if (Files.isDirectory(p)) {
|
||||
zos.putNextEntry(entry);
|
||||
DirectoryStream<Path> stream = Files.newDirectoryStream(p);
|
||||
for (Path path : stream) {
|
||||
addEntry(path,r,zos);
|
||||
addEntry(path,r,zos, prefix);
|
||||
}
|
||||
} else {
|
||||
entry.setTime(Files.getLastModifiedTime(p).toMillis());
|
@ -18,6 +18,8 @@ package io.nosqlbench.virtdata.library.basics.shared.from_string;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import io.nosqlbench.virtdata.api.annotations.Categories;
|
||||
import io.nosqlbench.virtdata.api.annotations.Category;
|
||||
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
|
||||
import org.apache.commons.text.StringEscapeUtils;
|
||||
|
||||
@ -38,6 +40,7 @@ import java.util.function.Function;
|
||||
* }</pre>
|
||||
*/
|
||||
@ThreadSafeMapper
|
||||
@Categories({Category.conversion, Category.general})
|
||||
public class EscapeJSON implements Function<String,String> {
|
||||
Gson gson = new GsonBuilder().create();
|
||||
|
||||
|
@ -31,12 +31,11 @@ import java.util.Arrays;
|
||||
public class VirtDataMainApp implements BundledApp {
|
||||
|
||||
private final static String APP_TESTMAPPER = "testmapper";
|
||||
private final static String APP_GENDOCS = "gendocs";
|
||||
private final static String APP_DIAGNOSE = "diagnose";
|
||||
private final static String[] names = new String[]{APP_GENDOCS, APP_TESTMAPPER, APP_DIAGNOSE};
|
||||
private final static String[] names = new String[]{APP_TESTMAPPER, APP_DIAGNOSE};
|
||||
|
||||
public static boolean hasNamedApp(String appname) {
|
||||
return (appname.equals(APP_TESTMAPPER) || appname.equals(APP_GENDOCS) || appname.equals(APP_DIAGNOSE));
|
||||
return (appname.equals(APP_TESTMAPPER) || appname.equals(APP_DIAGNOSE));
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
@ -46,7 +45,7 @@ public class VirtDataMainApp implements BundledApp {
|
||||
@Override
|
||||
public int applyAsInt(String[] args) {
|
||||
if (args.length == 0) {
|
||||
System.out.println("Usage: app (" + APP_TESTMAPPER + "|" + APP_GENDOCS + "|" + APP_DIAGNOSE +")");
|
||||
System.out.println("Usage: app (" + APP_TESTMAPPER +"|"+ APP_DIAGNOSE +")");
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -58,8 +57,6 @@ public class VirtDataMainApp implements BundledApp {
|
||||
|
||||
if (appSelection.equalsIgnoreCase(APP_TESTMAPPER)) {
|
||||
VirtDataCheckPerfApp.main(appArgs);
|
||||
} else if (appSelection.equalsIgnoreCase(APP_GENDOCS)) {
|
||||
VirtDataGenDocsApp.main(appArgs);
|
||||
} else if (appSelection.equalsIgnoreCase(APP_DIAGNOSE)) {
|
||||
VirtDataDiagnoseApp.main(appArgs);
|
||||
} else {
|
||||
|
@ -36,8 +36,9 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
public class VirtDataGenDocsApp implements Runnable {
|
||||
public class VirtDataGenDocsApp implements Callable<Map<String, StringBuilder>> {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(VirtDataGenDocsApp.class);
|
||||
|
||||
@ -55,6 +56,7 @@ public class VirtDataGenDocsApp implements Runnable {
|
||||
|
||||
private final String[] args;
|
||||
private final Map<String, Writer> writers = new HashMap<>();
|
||||
Map<String, StringBuilder> stringBuilders = new HashMap<String, StringBuilder>();
|
||||
|
||||
private String baseFileName = BASE_FILENAME;
|
||||
private String categories = CATEGORIES_SPLIT;
|
||||
@ -64,22 +66,21 @@ public class VirtDataGenDocsApp implements Runnable {
|
||||
private String basedir = "";
|
||||
|
||||
public static void main(String[] args) {
|
||||
new VirtDataGenDocsApp(args).run();
|
||||
new VirtDataGenDocsApp(args).call();
|
||||
}
|
||||
|
||||
public VirtDataGenDocsApp(String[] args) {
|
||||
this.args = args;
|
||||
}
|
||||
public VirtDataGenDocsApp(String[] args) {this.args = args;}
|
||||
|
||||
public void run() {
|
||||
LinkedList<String> largs = new LinkedList<>(Arrays.asList(args));
|
||||
public Map<String, StringBuilder> call() {
|
||||
|
||||
/*LinkedList<String> largs = new LinkedList<>(Arrays.asList(args));
|
||||
if (args.length > 0 && args[0].contains("help")) {
|
||||
System.out.println(
|
||||
"usage:\n" +
|
||||
"[basefile <name>] [basedir <dir>] [categories combined|split] [format json|markdown] " +
|
||||
"[blurbsdirs <dir>[:...]]\n\n"
|
||||
);
|
||||
return;
|
||||
return result;
|
||||
}
|
||||
while (largs.peekFirst() != null) {
|
||||
String argtype = largs.removeFirst();
|
||||
@ -112,12 +113,12 @@ public class VirtDataGenDocsApp implements Runnable {
|
||||
break;
|
||||
default:
|
||||
}
|
||||
}
|
||||
}*/
|
||||
|
||||
Optional<FDoc> docsinfo = loadAllDocs();
|
||||
|
||||
if (!docsinfo.isPresent()) {
|
||||
return;
|
||||
return stringBuilders;
|
||||
}
|
||||
|
||||
try {
|
||||
@ -131,38 +132,43 @@ public class VirtDataGenDocsApp implements Runnable {
|
||||
+ (this.categories.equals(CATEGORIES_SPLIT) ? "_" + categoryName : "")
|
||||
+ extension;
|
||||
|
||||
Writer writer = getWriterFor(filename);
|
||||
// Writer writer = getWriterFor(filename);
|
||||
StringBuilder builder = getBuilderFor(filename);
|
||||
String name = filename;
|
||||
|
||||
for (FDocFuncs docsForFuncName : docsForCatName) {
|
||||
if (format.equals(FORMAT_JSON)) {
|
||||
Gson gson = new GsonBuilder().setPrettyPrinting().create();
|
||||
writer.append(gson.toJson(docsForFuncName));
|
||||
builder.append(gson.toJson(docsForFuncName));
|
||||
} else if (format.equals(FORMAT_MARKDOWN)) {
|
||||
String markdown = docsForFuncName.asMarkdown();
|
||||
writer.append(markdown);
|
||||
builder.append(markdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Writer writer : writers.values()) {
|
||||
/*for (Writer writer : writers.values()) {
|
||||
writer.flush();
|
||||
writer.close();
|
||||
}
|
||||
}*/
|
||||
} catch (Exception e) {
|
||||
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return stringBuilders;
|
||||
}
|
||||
|
||||
private Writer getWriterFor(String outputname) {
|
||||
FileWriter fileWriter = null;
|
||||
if (!writers.containsKey(outputname)) {
|
||||
private StringBuilder getBuilderFor(String outputname) {
|
||||
// FileWriter fileWriter = null;
|
||||
StringBuilder builder = null;
|
||||
if (!stringBuilders.containsKey(outputname)) {
|
||||
try {
|
||||
outputname = basedir.isEmpty() ? outputname : basedir + "/" + outputname;
|
||||
Path parent = Path.of(outputname).getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
fileWriter = new FileWriter(outputname, false);
|
||||
writers.put(outputname, fileWriter);
|
||||
builder = new StringBuilder();
|
||||
stringBuilders.put(outputname, builder);
|
||||
|
||||
String[] blurbsdirs = blurbsDirs.split(":");
|
||||
for (String blurbsdir : blurbsdirs) {
|
||||
@ -173,7 +179,7 @@ public class VirtDataGenDocsApp implements Runnable {
|
||||
String blurb = Files.readString(blurbsFile, StandardCharsets.UTF_8);
|
||||
|
||||
logger.debug("writing blurb to " + outputname);
|
||||
fileWriter.append(blurb);
|
||||
builder.append(blurb);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -181,7 +187,7 @@ public class VirtDataGenDocsApp implements Runnable {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
return writers.get(outputname);
|
||||
return stringBuilders.get(outputname);
|
||||
}
|
||||
|
||||
private Optional<FDoc> loadAllDocs() {
|
||||
|
Loading…
Reference in New Issue
Block a user