diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a69a37e0b..68ffdd494 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,43 +2,105 @@ name: build on: push: + branches: + - main pull_request: jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 - name: checkout nosqlbench + - uses: actions/checkout@v3 + name: checkout nosqlbench + - uses: actions/setup-java@v3 + name: setup java + with: + java-version: '17' + java-package: jdk + architecture: x64 + distribution: 'temurin' - - uses: actions/setup-java@v3 - name: setup java - with: - java-version: '17' - java-package: jdk - architecture: x64 - distribution: 'temurin' + - name: Cache Maven packages + uses: actions/cache@v1 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 - - name: Cache Maven packages - uses: actions/cache@v1 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + - name: mvn-package + run: mvn package - - name: mvn-package - run: mvn package + - name: export docs + run: nb5/target/nb5 export-docs - - name: mvn-verify - run: mvn verify + - name: upload docs artifact + uses: actions/upload-artifact@v3 + with: + name: exported-docs + path: exported_docs.zip - - name: Capture - if: success() || failure() - run: tar -cvf logfiles.tar [a-zA-Z]**/logs/* + - name: mvn verify + run: mvn verify - - name: Archive Test Results - if: success() || failure() - uses: actions/upload-artifact@v3 - with: - name: test-results - path: logfiles.tar + - name: Capture + if: success() || failure() + run: tar -cvf logfiles.tar [a-zA-Z]**/logs/* + + - name: Archive Test Results + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: test-results + path: logfiles.tar + + + docs: + needs: build + runs-on: ubuntu-20.04 + steps: + - name: set git username + run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}" + + - name: set git email + run: git config --global user.name "${{ secrets.NBDROID_NAME }}" + + - name: download exported-docs + uses: actions/download-artifact@v3 + with: + name: exported-docs + + - name: unzip docs + run: unzip exported_docs.zip + + - run: ls -la + + - name: clone nosqlbench-build-docs + env: + NBDROID_NAME: ${{ secrets.NBDROID_NAME }} + NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }} + run: | + git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs + cd nosqlbench-build-docs + echo "files listing" + find . + git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git + git remote -v + + - name: push changes + env: + NBDROID_NAME: ${{ secrets.NBDROID_NAME }} + NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }} + run: | + rsync -av --delete -I --exclude '_index.md' drivers/ nosqlbench-build-docs/site/content/docs/drivers + rsync -av --delete -I --exclude '_index.md' bindings/ nosqlbench-build-docs/site/content/docs/bindings + echo "previewdocs.nosqlbench.io" > nosqlbench-build-docs/site/staticCNAME + cd nosqlbench-build-docs + git add -A + CHANGES=$(git status --porcelain 2>/dev/null| wc -l) + echo "found $CHANGES to push for doc updates" + if (( $CHANGES > 0 )) + then + git commit -m"docs update for $GITHUB_REF" + git push + fi + echo "push completed" diff --git a/driver-pulsar/src/main/resources/pulsar.md b/adapter-pulsar/src/main/resources/docs/pulsar.md similarity index 99% rename from driver-pulsar/src/main/resources/pulsar.md rename to adapter-pulsar/src/main/resources/docs/pulsar.md index f202c23d4..e74d86ede 100644 --- a/driver-pulsar/src/main/resources/pulsar.md +++ b/adapter-pulsar/src/main/resources/docs/pulsar.md @@ -1,3 +1,5 @@ +# Pulsar + - [1. Overview](#1-overview) - [1.1. Issues Tracker](#11-issues-tracker) - [2. NB Pulsar Driver Workload Definition Yaml File - High Level Structure](#2-nb-pulsar-driver-workload-definition-yaml-file---high-level-structure) diff --git a/adapter-pulsar/src/main/resources/pulsar.md b/adapter-pulsar/src/main/resources/pulsar.md index 17b57dc6e..99cc40934 100644 --- a/adapter-pulsar/src/main/resources/pulsar.md +++ b/adapter-pulsar/src/main/resources/pulsar.md @@ -1,3 +1,4 @@ +# Table of contents - [1. Overview](#1-overview) - [1.1. Issues Tracker](#11-issues-tracker) - [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload) diff --git a/adapter-s4j/src/main/resources/s4j.md b/adapter-s4j/src/main/resources/s4j.md index 9b35b8e71..f410ff116 100644 --- a/adapter-s4j/src/main/resources/s4j.md +++ b/adapter-s4j/src/main/resources/s4j.md @@ -1,3 +1,4 @@ +# S4J Adapter - [1. Overview](#1-overview) - [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload) - [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file) diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java index cfc55dfc2..d1f69307d 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java @@ -19,23 +19,27 @@ package io.nosqlbench.engine.api.activityimpl.uniform; import io.nosqlbench.api.docsapi.BundledMarkdownManifest; import io.nosqlbench.api.docsapi.Docs; import io.nosqlbench.api.docsapi.DocsBinder; +import io.nosqlbench.api.docsapi.DocsNameSpace; import io.nosqlbench.nb.annotations.Maturity; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.api.spi.SimpleServiceLoader; +import java.nio.file.Path; import java.util.List; +import java.util.Map; +import java.util.Set; -@Service(value = BundledMarkdownManifest.class, selector = "adapter-docs") +@Service(value = BundledMarkdownManifest.class, selector = "drivers") public class BundledDriverAdapterDocs implements BundledMarkdownManifest { @Override public DocsBinder getDocs() { - Docs docs = new Docs().namespace("adapter-docs"); + DocsBinder docs = new Docs(); SimpleServiceLoader loader = new SimpleServiceLoader<>(DriverAdapter.class, Maturity.Any); List> namedProviders = loader.getNamedProviders(); for (SimpleServiceLoader.Component namedProvider : namedProviders) { DriverAdapter driverAdapter = namedProvider.provider.get(); DocsBinder bundledDocs = driverAdapter.getBundledDocs(); - docs.merge(bundledDocs); + docs = docs.merge(bundledDocs); } return docs; } diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java index 14bfae422..2cb3a8005 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java @@ -28,11 +28,15 @@ import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.function.LongFunction; +import java.util.stream.Stream; /** *

The DriverAdapter interface is expected to be the replacement @@ -177,7 +181,7 @@ public interface DriverAdapter { * @return A {@link DocsBinder} which describes docs to include for a given adapter. */ default DocsBinder getBundledDocs() { - Docs docs = new Docs().namespace("adapter-"+this.getAdapterName()); + Docs docs = new Docs().namespace("drivers"); String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName(); String cp_docspath = "docs/" + this.getAdapterName(); @@ -185,6 +189,7 @@ public interface DriverAdapter { bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf); Optional> maindoc = NBIO.local().name("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first(); + maindoc.map(Content::asPath).ifPresent(docs::addPath); return docs.asDocsBinder(); diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarAction.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarAction.java deleted file mode 100644 index 329078544..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarAction.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar; - -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.pulsar.ops.PulsarOp; -import io.nosqlbench.engine.api.activityapi.core.SyncAction; -import io.nosqlbench.engine.api.activityapi.errorhandling.modular.ErrorDetail; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.function.LongFunction; - -public class PulsarAction implements SyncAction { - - private final static Logger logger = LogManager.getLogger(PulsarAction.class); - - private final int slot; - private final PulsarActivity activity; - int maxTries = 1; - - public PulsarAction(PulsarActivity activity, int slot) { - this.activity = activity; - this.slot = slot; - this.maxTries = activity.getActivityDef().getParams().getOptionalInteger("maxtries").orElse(10); - } - - @Override - public void init() { - } - - @Override - public int runCycle(long cycle) { - - // let's fail the action if some async operation failed - activity.failOnAsyncOperationFailure(); - - long start = System.nanoTime(); - - PulsarOp pulsarOp; - try (Timer.Context ctx = activity.getBindTimer().time()) { - LongFunction readyPulsarOp = activity.getSequencer().apply(cycle); - pulsarOp = readyPulsarOp.apply(cycle); - } catch (Exception bindException) { - // if diagnostic mode ... - activity.getErrorHandler().handleError(bindException, cycle, 0); - throw new RuntimeException( - "while binding request in cycle " + cycle + ": " + bindException.getMessage(), bindException - ); - } - - for (int i = 0; i < maxTries; i++) { - Timer.Context ctx = activity.getExecuteTimer().time(); - try { - // it is up to the pulsarOp to call Context#close when the activity is executed - // this allows us to track time for async operations - pulsarOp.run(ctx::close); - break; - } catch (RuntimeException err) { - ErrorDetail errorDetail = activity - .getErrorHandler() - .handleError(err, cycle, System.nanoTime() - start); - if (!errorDetail.isRetryable()) { - break; - } - } - } - - return 0; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java deleted file mode 100644 index 0a487ce3a..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivity.java +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.pulsar.ops.PulsarOp; -import io.nosqlbench.driver.pulsar.ops.ReadyPulsarOp; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import io.nosqlbench.driver.pulsar.util.PulsarNBClientConf; -import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver; -import io.nosqlbench.engine.api.activityapi.errorhandling.modular.NBErrorHandler; -import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter; -import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.engine.api.activityimpl.OpDispenser; -import io.nosqlbench.engine.api.activityimpl.SimpleActivity; -import io.nosqlbench.api.engine.metrics.ActivityMetrics; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminBuilder; -import org.apache.pulsar.client.api.*; -import org.apache.pulsar.common.schema.KeyValueEncodingType; - -import java.util.Map; -import java.util.Optional; - -public class PulsarActivity extends SimpleActivity implements ActivityDefObserver { - - private final static Logger logger = LogManager.getLogger(PulsarActivity.class); - - private Counter bytesCounter; - private Histogram messageSizeHistogram; - private Timer bindTimer; - private Timer executeTimer; - private Timer createTransactionTimer; - private Timer commitTransactionTimer; - - // Metrics for NB Pulsar driver milestone: https://github.com/nosqlbench/nosqlbench/milestone/11 - // - end-to-end latency - private Histogram e2eMsgProcLatencyHistogram; - - /** - * A histogram that tracks payload round-trip-time, based on a user-defined field in some sender - * system which can be interpreted as millisecond epoch time in the system's local time zone. - * This is paired with a field name of the same type to be extracted and reported in a meteric - * named 'payload-rtt'. - */ - private Histogram payloadRttHistogram; - - // - message out of sequence error counter - private Counter msgErrOutOfSeqCounter; - // - message loss counter - private Counter msgErrLossCounter; - // - message duplicate (when dedup is enabled) error counter - private Counter msgErrDuplicateCounter; - - private PulsarSpaceCache pulsarCache; - - private PulsarNBClientConf pulsarNBClientConf; - private String pulsarSvcUrl; - private String webSvcUrl; - private PulsarAdmin pulsarAdmin; - private PulsarClient pulsarClient; - private Schema pulsarSchema; - - private NBErrorHandler errorHandler; - private OpSequence> sequencer; - private volatile Throwable asyncOperationFailure; - private boolean cycleratePerThread; - - public PulsarActivity(ActivityDef activityDef) { - super(activityDef); - } - - @Override - public void shutdownActivity() { - super.shutdownActivity(); - - if (pulsarCache == null) { - return; - } - - for (PulsarSpace pulsarSpace : pulsarCache.getAssociatedPulsarSpace()) { - pulsarSpace.shutdownPulsarSpace(); - } - } - - @Override - public void initActivity() { - super.initActivity(); - pulsarCache = new PulsarSpaceCache(this); - - bytesCounter = ActivityMetrics.counter(activityDef, "bytes"); - messageSizeHistogram = ActivityMetrics.histogram(activityDef, "message_size", this.getHdrDigits()); - bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits()); - executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits()); - createTransactionTimer = ActivityMetrics.timer(activityDef, "create_transaction", this.getHdrDigits()); - commitTransactionTimer = ActivityMetrics.timer(activityDef, "commit_transaction", this.getHdrDigits()); - - e2eMsgProcLatencyHistogram = ActivityMetrics.histogram(activityDef, "e2e_msg_latency", this.getHdrDigits()); - payloadRttHistogram = ActivityMetrics.histogram(activityDef, "payload_rtt", this.getHdrDigits()); - - msgErrOutOfSeqCounter = ActivityMetrics.counter(activityDef, "err_msg_oos"); - msgErrLossCounter = ActivityMetrics.counter(activityDef, "err_msg_loss"); - msgErrDuplicateCounter = ActivityMetrics.counter(activityDef, "err_msg_dup"); - - String pulsarClntConfFile = - activityDef.getParams().getOptionalString("config").orElse("config.properties"); - pulsarNBClientConf = new PulsarNBClientConf(pulsarClntConfFile); - - pulsarSvcUrl = - activityDef.getParams().getOptionalString("service_url").orElse("pulsar://localhost:6650"); - webSvcUrl = - activityDef.getParams().getOptionalString("web_url").orElse("http://localhost:8080"); - - initPulsarAdminAndClientObj(); - createPulsarSchemaFromConf(); - - - this.sequencer = createOpSequence((ot) -> new ReadyPulsarOp(ot, pulsarCache, this), false, Optional.empty()); - setDefaultsFromOpSequence(sequencer); - onActivityDefUpdate(activityDef); - - this.errorHandler = new NBErrorHandler( - () -> activityDef.getParams().getOptionalString("errors").orElse("stop"), - this::getExceptionMetrics - ); - - cycleratePerThread = activityDef.getParams().takeBoolOrDefault("cyclerate_per_thread", false); - } - - private final ThreadLocal cycleLimiterThreadLocal = ThreadLocal.withInitial(() -> { - if (super.getCycleLimiter() != null) { - return RateLimiters.createOrUpdate(this.getActivityDef(), "cycles", null, - super.getCycleLimiter().getRateSpec()); - } else { - return null; - } - }); - - @Override - public RateLimiter getCycleLimiter() { - if (cycleratePerThread) { - return cycleLimiterThreadLocal.get(); - } else { - return super.getCycleLimiter(); - } - } - - public NBErrorHandler getErrorHandler() { return errorHandler; } - - public OpSequence> getSequencer() { return sequencer; } - - public void failOnAsyncOperationFailure() { - if (asyncOperationFailure != null) { - throw new RuntimeException(asyncOperationFailure); - } - } - - public void asyncOperationFailed(Throwable ex) { - this.asyncOperationFailure = ex; - } - - /** - * Initialize - * - PulsarAdmin object for adding/deleting tenant, namespace, and topic - * - PulsarClient object for message publishing and consuming - */ - private void initPulsarAdminAndClientObj() { - PulsarAdminBuilder adminBuilder = - PulsarAdmin.builder() - .serviceHttpUrl(webSvcUrl); - - ClientBuilder clientBuilder = PulsarClient.builder(); - - try { - Map clientConfMap = pulsarNBClientConf.getClientConfMap(); - - // Override "client.serviceUrl" setting in config.properties - clientConfMap.remove("serviceUrl"); - clientBuilder.loadConf(clientConfMap).serviceUrl(pulsarSvcUrl); - - // Pulsar Authentication - String authPluginClassName = - (String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.authPulginClassName.label); - String authParams = - (String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.authParams.label); - - if ( !StringUtils.isAnyBlank(authPluginClassName, authParams) ) { - adminBuilder.authentication(authPluginClassName, authParams); - clientBuilder.authentication(authPluginClassName, authParams); - } - - String useTlsStr = - (String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.useTls.label); - boolean useTls = BooleanUtils.toBoolean(useTlsStr); - - String tlsTrustCertsFilePath = - (String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsTrustCertsFilePath.label); - - String tlsAllowInsecureConnectionStr = - (String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsAllowInsecureConnection.label); - boolean tlsAllowInsecureConnection = BooleanUtils.toBoolean(tlsAllowInsecureConnectionStr); - - String tlsHostnameVerificationEnableStr = - (String) pulsarNBClientConf.getClientConfValue(PulsarActivityUtil.CLNT_CONF_KEY.tlsHostnameVerificationEnable.label); - boolean tlsHostnameVerificationEnable = BooleanUtils.toBoolean(tlsHostnameVerificationEnableStr); - - if ( useTls ) { - adminBuilder - .enableTlsHostnameVerification(tlsHostnameVerificationEnable); - - clientBuilder - .enableTlsHostnameVerification(tlsHostnameVerificationEnable); - - if (!StringUtils.isBlank(tlsTrustCertsFilePath)) { - adminBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); - clientBuilder.tlsTrustCertsFilePath(tlsTrustCertsFilePath); - } - } - - // Put this outside "if (useTls)" block for easier handling of "tlsAllowInsecureConnection" - adminBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); - clientBuilder.allowTlsInsecureConnection(tlsAllowInsecureConnection); - - pulsarAdmin = adminBuilder.build(); - pulsarClient = clientBuilder.build(); - - //////////////// - // Not supported in Pulsar 2.8.0 - // - // ClientConfigurationData configurationData = pulsarAdmin.getClientConfigData(); - // logger.debug(configurationData.toString()); - - } catch (PulsarClientException e) { - logger.error("Fail to create PulsarAdmin and/or PulsarClient object from the global configuration!"); - throw new RuntimeException("Fail to create PulsarAdmin and/or PulsarClient object from global configuration!"); - } - } - - /** - * Get Pulsar schema from the definition string - */ - private void createPulsarSchemaFromConf() { - pulsarSchema = buldSchemaFromDefinition("schema.type", "schema.definition"); - - // this is to allow KEY_VALUE schema - if (pulsarNBClientConf.hasSchemaConfKey("schema.key.type")) { - Schema pulsarKeySchema = buldSchemaFromDefinition("schema.key.type", "schema.key.definition"); - Object encodingType = pulsarNBClientConf.getSchemaConfValue("schema.keyvalue.encodingtype"); - KeyValueEncodingType keyValueEncodingType = KeyValueEncodingType.SEPARATED; - if (encodingType != null) { - keyValueEncodingType = KeyValueEncodingType.valueOf(encodingType.toString()); - } - pulsarSchema = Schema.KeyValue(pulsarKeySchema, pulsarSchema, keyValueEncodingType); - } - } - - private Schema buldSchemaFromDefinition(String schemaTypeConfEntry, - String schemaDefinitionConfEntry) { - Object value = pulsarNBClientConf.getSchemaConfValue(schemaTypeConfEntry); - Object schemaDefinition = pulsarNBClientConf.getSchemaConfValue(schemaDefinitionConfEntry); - String schemaType = (value != null) ? value.toString() : ""; - - Schema result; - if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType)) { - String schemaDefStr = (schemaDefinition != null) ? schemaDefinition.toString() : ""; - result = PulsarActivityUtil.getAvroSchema(schemaType, schemaDefStr); - } else if (PulsarActivityUtil.isPrimitiveSchemaTypeStr(schemaType)) { - result = PulsarActivityUtil.getPrimitiveTypeSchema(schemaType); - } else if (PulsarActivityUtil.isAutoConsumeSchemaTypeStr(schemaType)) { - result = Schema.AUTO_CONSUME(); - } else { - throw new RuntimeException("Unsupported schema type string: " + schemaType + "; " + - "Only primitive type, Avro type and AUTO_CONSUME are supported at the moment!"); - } - return result; - } - - public PulsarNBClientConf getPulsarConf() { return this.pulsarNBClientConf;} - public String getPulsarSvcUrl() { return this.pulsarSvcUrl;} - public String getWebSvcUrl() { return this.webSvcUrl; } - public PulsarAdmin getPulsarAdmin() { return this.pulsarAdmin; } - public PulsarClient getPulsarClient() { return this.pulsarClient; } - public Schema getPulsarSchema() { return pulsarSchema; } - - public Counter getBytesCounter() { return bytesCounter; } - public Histogram getMessageSizeHistogram() { return messageSizeHistogram; } - public Timer getBindTimer() { return bindTimer; } - public Timer getExecuteTimer() { return this.executeTimer; } - public Timer getCreateTransactionTimer() { return createTransactionTimer; } - public Timer getCommitTransactionTimer() { return commitTransactionTimer; } - - public Histogram getPayloadRttHistogram() {return payloadRttHistogram;} - public Histogram getE2eMsgProcLatencyHistogram() { return e2eMsgProcLatencyHistogram; } - public Counter getMsgErrOutOfSeqCounter() { return msgErrOutOfSeqCounter; } - public Counter getMsgErrLossCounter() { return msgErrLossCounter; } - public Counter getMsgErrDuplicateCounter() { return msgErrDuplicateCounter; } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivityType.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivityType.java deleted file mode 100644 index 281f0092b..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarActivityType.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar; - -import io.nosqlbench.engine.api.activityapi.core.Action; -import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; -import io.nosqlbench.engine.api.activityapi.core.ActivityType; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.nb.annotations.Service; - -@Service(value= ActivityType.class, selector="pulsar") -public class PulsarActivityType implements ActivityType { - - @Override - public ActionDispenser getActionDispenser(PulsarActivity activity) { - if (activity.getParams().getOptionalString("async").isPresent()) { - throw new RuntimeException("The async pulsar driver is not implemented yet."); - } - return new PulsarActionDispenser(activity); - } - - @Override - public PulsarActivity getActivity(ActivityDef activityDef) { - return new PulsarActivity(activityDef); - } - - private static class PulsarActionDispenser implements ActionDispenser { - private final PulsarActivity activity; - public PulsarActionDispenser(PulsarActivity activity) { - this.activity = activity; - } - - @Override - public Action getAction(int slot) { - return new PulsarAction(activity, slot); - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpace.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpace.java deleted file mode 100644 index e8c4502bf..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpace.java +++ /dev/null @@ -1,802 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import io.nosqlbench.driver.pulsar.util.PulsarNBClientConf; -import io.nosqlbench.api.engine.activityimpl.ActivityDef; -import io.nosqlbench.api.engine.metrics.ActivityMetrics; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.admin.Clusters; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.*; -import org.apache.pulsar.client.api.transaction.Transaction; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -/** - * An instance of a pulsar client, along with all the cached objects which are normally - * associated with it during a client session in a typical application. - * A PulsarSpace is simply a named and cached set of objects which must be used together. - */ -public class PulsarSpace { - - private final static Logger logger = LogManager.getLogger(PulsarSpace.class); - - private final String spaceName; - - private final ConcurrentHashMap> producers = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> consumers = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> readers = new ConcurrentHashMap<>(); - - private final PulsarActivity pulsarActivity; - private final ActivityDef activityDef; - - private final PulsarNBClientConf pulsarNBClientConf; - private final String pulsarSvcUrl; - private final String webSvcUrl; - private final PulsarAdmin pulsarAdmin; - private final PulsarClient pulsarClient; - private final Schema pulsarSchema; - private final Set pulsarClusterMetadata = new HashSet<>(); - private final Timer createTransactionTimer; - - public PulsarSpace(String name, PulsarActivity pulsarActivity) { - this.spaceName = name; - this.pulsarActivity = pulsarActivity; - - this.pulsarNBClientConf = pulsarActivity.getPulsarConf(); - this.pulsarSvcUrl = pulsarActivity.getPulsarSvcUrl(); - this.webSvcUrl = pulsarActivity.getWebSvcUrl(); - this.pulsarAdmin = pulsarActivity.getPulsarAdmin(); - this.pulsarClient = pulsarActivity.getPulsarClient(); - this.pulsarSchema = pulsarActivity.getPulsarSchema(); - this.activityDef = pulsarActivity.getActivityDef(); - this.createTransactionTimer = pulsarActivity.getCreateTransactionTimer(); - - try { - Clusters clusters = pulsarAdmin.clusters(); - List stringList = clusters.getClusters(); - CollectionUtils.addAll(pulsarClusterMetadata, stringList.listIterator()); - - } catch (PulsarAdminException e) { - // this is okay if you are connecting with a token that does not have access to the - // system configuration - logger.info("Could not get list of Pulsar Clusters from global configuration: " + e.getMessage()); - } - } - - public PulsarNBClientConf getPulsarClientConf() { return pulsarNBClientConf; } - public PulsarAdmin getPulsarAdmin() { return pulsarAdmin; } - public PulsarClient getPulsarClient() { return pulsarClient; } - public Schema getPulsarSchema() { return pulsarSchema; } - public String getPulsarSvcUrl() { return pulsarSvcUrl;} - public String getWebSvcUrl() { return webSvcUrl; } - public Set getPulsarClusterMetadata() { return pulsarClusterMetadata; } - - - // Properly shut down all Pulsar objects (producers, consumers, etc.) that are associated with this space - public void shutdownPulsarSpace() { - try { - for (Producer producer : producers.values()) { - if (producer != null) producer.close(); - } - - for (Consumer consumer : consumers.values()) { - if (consumer != null) consumer.close(); - } - - for (Reader reader : readers.values()) { - if (reader != null) reader.close(); - } - - if (pulsarAdmin != null) pulsarAdmin.close(); - - if (pulsarClient != null) pulsarClient.close(); - } - catch (Exception e) { - throw new RuntimeException("Unexpected error when closing Pulsar objects!"); - } - } - - /** - * Get a proper Pulsar API metrics prefix depending on the API type - * - * @param apiType - Pulsar API type: producer, consumer, reader, etc. - * @param apiObjName - actual name of a producer, a consumer, a reader, etc. - * @param topicName - topic name - * @return String - */ - private String getPulsarAPIMetricsPrefix(String apiType, String apiObjName, String topicName) { - String apiMetricsPrefix; - - if (!PulsarActivityUtil.isValidPulsarApiType(apiType)) { - throw new RuntimeException( - "Incorrect Pulsar API type. Valid type list: " + PulsarActivityUtil.getValidPulsarApiTypeList()); - } - - if (!StringUtils.isBlank(apiObjName)) { - apiMetricsPrefix = apiObjName + "_"; - } - else { - // we want a meaningful name for the API object (producer, consumer, reader, etc.) - // we are not appending the topic name - apiMetricsPrefix = apiType; - - if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label)) - apiMetricsPrefix += producers.size(); - else if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.CONSUMER.label)) - apiMetricsPrefix += consumers.size(); - else if (apiType.equalsIgnoreCase(PulsarActivityUtil.PULSAR_API_TYPE.READER.label)) - apiMetricsPrefix += readers.size(); - - apiMetricsPrefix += "_"; - } - - apiMetricsPrefix += topicName + "_"; - apiMetricsPrefix = apiMetricsPrefix - // default name for tests/demos (in all Pulsar examples) is persistent://public/default/test -> use just the topic name test - .replace("persistent://public/default/", "") - // always remove topic type - .replace("non-persistent://", "") - .replace("persistent://", "") - // persistent://tenant/namespace/topicname -> tenant_namespace_topicname - .replace("/","_"); - - return apiMetricsPrefix; - } - - - ////////////////////////////////////// - // Producer Processing --> start - ////////////////////////////////////// - // - private static class ProducerGaugeImpl implements Gauge { - private final Producer producer; - private final Function valueExtractor; - - ProducerGaugeImpl(Producer producer, Function valueExtractor) { - this.producer = producer; - this.valueExtractor = valueExtractor; - } - - @Override - public Object getValue() { - // see Pulsar bug https://github.com/apache/pulsar/issues/10100 - // we need to synchronize on producer otherwise we could receive corrupted data - synchronized(producer) { - return valueExtractor.apply(producer.getStats()); - } - } - } - static Gauge producerSafeExtractMetric(Producer producer, Function valueExtractor) { - return new ProducerGaugeImpl(producer, valueExtractor); - } - - // Producer name is NOT mandatory - // - It can be set at either global level or cycle level - // - If set at both levels, cycle level setting takes precedence - private String getEffectiveProducerName(String cycleProducerName) { - if (!StringUtils.isBlank(cycleProducerName)) { - return cycleProducerName; - } - - String globalProducerName = pulsarNBClientConf.getProducerName(); - if (!StringUtils.isBlank(globalProducerName)) { - return globalProducerName; - } - - return ""; - } - - public Supplier getTransactionSupplier() { - PulsarClient pulsarClient = getPulsarClient(); - return () -> { - try (Timer.Context time = createTransactionTimer.time() ){ - return pulsarClient - .newTransaction() - .build() - .get(); - } catch (ExecutionException | InterruptedException err) { - if (logger.isWarnEnabled()) { - logger.warn("Error while starting a new transaction", err); - } - throw new RuntimeException(err); - } catch (PulsarClientException err) { - throw new RuntimeException("Transactions are not enabled on Pulsar Client, " + - "please set client.enableTransaction=true in your Pulsar Client configuration"); - } - }; - } - - // Topic name IS mandatory - // - It must be set at either global level or cycle level - // - If set at both levels, cycle level setting takes precedence - private String getEffectiveProducerTopicName(String cycleTopicName) { - if (!StringUtils.isBlank(cycleTopicName)) { - return cycleTopicName; - } - - String globalTopicName = pulsarNBClientConf.getProducerTopicName(); - if (!StringUtils.isBlank(globalTopicName)) { - return globalTopicName; - } - - throw new RuntimeException("Producer topic name must be set at either global level or cycle level!"); - } - - public Producer getProducer(String cycleTopicName, String cycleProducerName) { - String topicName = getEffectiveProducerTopicName(cycleTopicName); - String producerName = getEffectiveProducerName(cycleProducerName); - - if (StringUtils.isBlank(topicName)) { - throw new RuntimeException("Producer:: must specify a topic name"); - } - - String producerCacheKey = PulsarActivityUtil.buildCacheKey(producerName, topicName); - Producer producer = producers.get(producerCacheKey); - - if (producer == null) { - PulsarClient pulsarClient = getPulsarClient(); - - // Get other possible producer settings that are set at global level - Map producerConf = pulsarNBClientConf.getProducerConfMap(); - - // Remove global level settings: "topicName" and "producerName" - producerConf.remove(PulsarActivityUtil.PRODUCER_CONF_STD_KEY.topicName.label); - producerConf.remove(PulsarActivityUtil.PRODUCER_CONF_STD_KEY.producerName.label); - - String producerMetricsPrefix = getPulsarAPIMetricsPrefix( - PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label, - producerName, - topicName); - - try { - ProducerBuilder producerBuilder = pulsarClient. - newProducer(pulsarSchema). - loadConf(producerConf). - topic(topicName); - - if (!StringUtils.isAnyBlank(producerName)) { - producerBuilder = producerBuilder.producerName(producerName); - } - - producer = producerBuilder.create(); - producers.put(producerCacheKey, producer); - - ActivityMetrics.gauge(activityDef, - producerMetricsPrefix + "total_bytes_sent", - producerSafeExtractMetric(producer, (s -> s.getTotalBytesSent() + s.getNumBytesSent()))); - ActivityMetrics.gauge(activityDef, - producerMetricsPrefix + "total_msg_sent", - producerSafeExtractMetric(producer, (s -> s.getTotalMsgsSent() + s.getNumMsgsSent()))); - ActivityMetrics.gauge(activityDef, - producerMetricsPrefix + "total_send_failed", - producerSafeExtractMetric(producer, (s -> s.getTotalSendFailed() + s.getNumSendFailed()))); - ActivityMetrics.gauge(activityDef, - producerMetricsPrefix + "total_ack_received", - producerSafeExtractMetric(producer,(s -> s.getTotalAcksReceived() + s.getNumAcksReceived()))); - ActivityMetrics.gauge(activityDef, - producerMetricsPrefix + "send_bytes_rate", - producerSafeExtractMetric(producer, ProducerStats::getSendBytesRate)); - ActivityMetrics.gauge(activityDef, - producerMetricsPrefix + "send_msg_rate", - producerSafeExtractMetric(producer, ProducerStats::getSendMsgsRate)); - } - catch (PulsarClientException ple) { - throw new RuntimeException("Unable to create a Pulsar producer!", ple); - } - } - - return producer; - } - // - ////////////////////////////////////// - // Producer Processing <-- end - ////////////////////////////////////// - - - ////////////////////////////////////// - // Consumer Processing --> start - ////////////////////////////////////// - // - private static class ConsumerGaugeImpl implements Gauge { - private final Consumer consumer; - private final Function valueExtractor; - - ConsumerGaugeImpl(Consumer consumer, Function valueExtractor) { - this.consumer = consumer; - this.valueExtractor = valueExtractor; - } - - @Override - public Object getValue() { - // see Pulsar bug https://github.com/apache/pulsar/issues/10100 - // - this is a bug report for producer stats. - // - assume this also applies to consumer stats. - synchronized(consumer) { - return valueExtractor.apply(consumer.getStats()); - } - } - } - static Gauge consumerSafeExtractMetric(Consumer consumer, Function valueExtractor) { - return new ConsumerGaugeImpl(consumer, valueExtractor); - } - - private String getEffectiveSubscriptionName(String cycleSubscriptionName) { - if (!StringUtils.isBlank(cycleSubscriptionName)) { - return cycleSubscriptionName; - } - - String globalSubscriptionName = pulsarNBClientConf.getConsumerSubscriptionName(); - if (!StringUtils.isBlank(globalSubscriptionName)) { - return globalSubscriptionName; - } - - throw new RuntimeException("Consumer::Subscription name must be set at either global level or cycle level!"); - } - - private String getEffectiveSubscriptionTypeStr(String cycleSubscriptionType) { - if (!StringUtils.isBlank(cycleSubscriptionType)) { - return cycleSubscriptionType; - } - - String globalSubscriptionType = pulsarNBClientConf.getConsumerSubscriptionType(); - if (!StringUtils.isBlank(globalSubscriptionType)) { - return globalSubscriptionType; - } - - return ""; - } - private SubscriptionType getEffectiveSubscriptionType(String cycleSubscriptionType) { - String effectiveSubscriptionStr = getEffectiveSubscriptionTypeStr(cycleSubscriptionType); - SubscriptionType subscriptionType = SubscriptionType.Exclusive; - - if (!StringUtils.isBlank(effectiveSubscriptionStr)) { - if (!PulsarActivityUtil.isValidSubscriptionType(effectiveSubscriptionStr)) { - throw new RuntimeException("Consumer::Invalid subscription type (\"" + - effectiveSubscriptionStr + "\"). \nValid subscription types: " + PulsarActivityUtil.getValidSubscriptionTypeList()); - } else { - subscriptionType = SubscriptionType.valueOf(effectiveSubscriptionStr); - } - } - - return subscriptionType; - } - - private String getEffectiveConsumerName(String cycleConsumerName) { - if (!StringUtils.isBlank(cycleConsumerName)) { - return cycleConsumerName; - } - - String globalConsumerName = pulsarNBClientConf.getConsumerName(); - if (!StringUtils.isBlank(globalConsumerName)) { - return globalConsumerName; - } - - return ""; - } - - public Consumer getConsumer(String cycleTopicName, - String cycleSubscriptionName, - String cycleSubscriptionType, - String cycleConsumerName, - String cycleKeySharedSubscriptionRanges) { - String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName); - SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType); - String consumerName = getEffectiveConsumerName(cycleConsumerName); - - if (StringUtils.isAnyBlank(cycleTopicName, subscriptionName)) { - throw new RuntimeException("Consumer:: must specify a topic name and a subscription name"); - } - - String consumerCacheKey = PulsarActivityUtil.buildCacheKey(consumerName, subscriptionName, cycleTopicName); - Consumer consumer = consumers.get(consumerCacheKey); - - if (consumer == null) { - PulsarClient pulsarClient = getPulsarClient(); - - // Get other possible consumer settings that are set at global level - Map consumerConf = new HashMap<>(pulsarNBClientConf.getConsumerConfMap()); - - // Remove global level settings: - // - "topicNames", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName" - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label); - // Remove non-standard consumer configuration properties - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); - - try { - ConsumerBuilder consumerBuilder = pulsarClient. - newConsumer(pulsarSchema). - loadConf(consumerConf). - topic(cycleTopicName). - subscriptionName(subscriptionName). - subscriptionType(subscriptionType); - - if (subscriptionType == SubscriptionType.Key_Shared) { - KeySharedPolicy keySharedPolicy = KeySharedPolicy.autoSplitHashRange(); - if (cycleKeySharedSubscriptionRanges != null && !cycleKeySharedSubscriptionRanges.isEmpty()) { - Range[] ranges = parseRanges(cycleKeySharedSubscriptionRanges); - logger.info("Configuring KeySharedPolicy#stickyHashRange with ranges {}", ranges); - keySharedPolicy = KeySharedPolicy.stickyHashRange().ranges(ranges); - } - consumerBuilder.keySharedPolicy(keySharedPolicy); - } - - if (!StringUtils.isBlank(consumerName)) { - consumerBuilder = consumerBuilder.consumerName(consumerName); - } - - consumer = consumerBuilder.subscribe(); - - String consumerMetricsPrefix = getPulsarAPIMetricsPrefix( - PulsarActivityUtil.PULSAR_API_TYPE.CONSUMER.label, - consumerName, - cycleTopicName); - - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "total_bytes_recv", - consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "total_msg_recv", - consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "total_recv_failed", - consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "total_acks_sent", - consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "recv_bytes_rate", - consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived)); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "recv_msg_rate", - consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived)); - } catch (PulsarClientException ple) { - ple.printStackTrace(); - throw new RuntimeException("Unable to create a Pulsar consumer!"); - } - - consumers.put(consumerCacheKey, consumer); - } - - return consumer; - } - - private static Range[] parseRanges(String ranges) { - if (ranges == null || ranges.isEmpty()) { - return new Range[0]; - } - String[] split = ranges.split(","); - Range[] result = new Range[split.length]; - for (int i = 0; i < split.length; i++) { - String range = split[i]; - int pos = range.indexOf(".."); - if (pos <= 0) { - throw new IllegalArgumentException("Invalid range '" + range + "'"); - } - try { - int start = Integer.parseInt(range.substring(0, pos)); - int end = Integer.parseInt(range.substring(pos + 2)); - result[i] = Range.of(start, end); - } catch (NumberFormatException err) { - throw new IllegalArgumentException("Invalid range '" + range + "'"); - } - } - return result; - } - - // - ////////////////////////////////////// - // Consumer Processing <-- end - ////////////////////////////////////// - - - ////////////////////////////////////// - // Multi-topic Consumer Processing --> start - ////////////////////////////////////// - // - private String getEffectiveConsumerTopicNameListStr(String cycleTopicNames) { - if (!StringUtils.isBlank(cycleTopicNames)) { - return cycleTopicNames; - } - - String globalTopicNames = pulsarNBClientConf.getConsumerTopicNames(); - if (!StringUtils.isBlank(globalTopicNames)) { - return globalTopicNames; - } - - return ""; - } - - private List getEffectiveConsumerTopicNameList(String cycleTopicNames) { - String effectiveTopicNamesStr = getEffectiveConsumerTopicNameListStr(cycleTopicNames); - - String[] names = effectiveTopicNamesStr.split("[;,]"); - ArrayList effectiveTopicNameList = new ArrayList<>(); - - for (String name : names) { - if (!StringUtils.isBlank(name)) - effectiveTopicNameList.add(name.trim()); - } - - return effectiveTopicNameList; - } - - private String getEffectiveConsumerTopicPatternStr(String cycleTopicsPattern) { - if (!StringUtils.isBlank(cycleTopicsPattern)) { - return cycleTopicsPattern; - } - - String globalTopicsPattern = pulsarNBClientConf.getConsumerTopicPattern(); - if (!StringUtils.isBlank(globalTopicsPattern)) { - return globalTopicsPattern; - } - - return ""; - } - - private Pattern getEffectiveConsumerTopicPattern(String cycleTopicsPattern) { - String effectiveTopicsPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicsPattern); - Pattern topicsPattern; - try { - if (!StringUtils.isBlank(effectiveTopicsPatternStr)) - topicsPattern = Pattern.compile(effectiveTopicsPatternStr); - else - topicsPattern = null; - } catch (PatternSyntaxException pse) { - topicsPattern = null; - } - return topicsPattern; - } - - public Consumer getMultiTopicConsumer( - String cycleTopicUri, - String cycleTopicNameList, - String cycleTopicsPattern, - String cycleSubscriptionName, - String cycleSubscriptionType, - String cycleConsumerName) { - - List topicNameList = getEffectiveConsumerTopicNameList(cycleTopicNameList); - String topicsPatternStr = getEffectiveConsumerTopicPatternStr(cycleTopicsPattern); - Pattern topicsPattern = getEffectiveConsumerTopicPattern(cycleTopicsPattern); - String subscriptionName = getEffectiveSubscriptionName(cycleSubscriptionName); - SubscriptionType subscriptionType = getEffectiveSubscriptionType(cycleSubscriptionType); - String consumerName = getEffectiveConsumerName(cycleConsumerName); - - if ( subscriptionType.equals(SubscriptionType.Exclusive) && (activityDef.getThreads() > 1) ) { - throw new RuntimeException("Consumer:: trying to create multiple consumers of " + - "\"Exclusive\" subscription type under the same subscription name to the same topic!"); - } - - if (StringUtils.isBlank(cycleTopicUri) && topicNameList.isEmpty() && (topicsPattern == null)) { - throw new RuntimeException("Consumer:: \"topic_uri\", \"topic_names\" and \"topics_pattern\" parameters can't be all empty/invalid!"); - } - - // precedence sequence: - // topic_names (consumer statement param) > - // topics_pattern (consumer statement param) > - // topic_uri (document level param) - String consumerTopicListString; - if (!topicNameList.isEmpty()) { - consumerTopicListString = String.join("|", topicNameList); - } else if (topicsPattern != null) { - consumerTopicListString = topicsPatternStr; - } else { - consumerTopicListString = cycleTopicUri; - } - String consumerCacheKey = PulsarActivityUtil.buildCacheKey( - consumerName, - subscriptionName, - consumerTopicListString); - - Consumer consumer = consumers.get(consumerCacheKey); - - if (consumer == null) { - PulsarClient pulsarClient = getPulsarClient(); - - // Get other possible producer settings that are set at global level - Map consumerConf = new HashMap<>(pulsarNBClientConf.getConsumerConfMap()); - - // Remove global level settings: - // - "topicNameList", "topicsPattern", "subscriptionName", "subscriptionType", "consumerName" - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label); - // Remove non-standard consumer configuration properties - consumerConf.remove(PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); - - try { - ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(pulsarSchema). - loadConf(consumerConf). - subscriptionName(subscriptionName). - subscriptionType(subscriptionType). - consumerName(consumerName); - - if (!topicNameList.isEmpty()) { - consumerBuilder = consumerBuilder.topics(topicNameList); - } else if (topicsPattern != null) { - consumerBuilder = consumerBuilder.topicsPattern(topicsPattern); - } else { - consumerBuilder = consumerBuilder.topic(cycleTopicUri); - } - - consumer = consumerBuilder.subscribe(); - - String consumerMetricsPrefix = getPulsarAPIMetricsPrefix( - PulsarActivityUtil.PULSAR_API_TYPE.PRODUCER.label, - consumerName, - consumerTopicListString); - - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "totalBytesRecvd", - consumerSafeExtractMetric(consumer, (s -> s.getTotalBytesReceived() + s.getNumBytesReceived()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "totalMsgsRecvd", - consumerSafeExtractMetric(consumer, (s -> s.getTotalMsgsReceived() + s.getNumMsgsReceived()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "totalRecvdFailed", - consumerSafeExtractMetric(consumer, (s -> s.getTotalReceivedFailed() + s.getNumReceiveFailed()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "totalAcksSent", - consumerSafeExtractMetric(consumer,(s -> s.getTotalAcksSent() + s.getNumAcksSent()))); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "recvdBytesRate", - consumerSafeExtractMetric(consumer, ConsumerStats::getRateBytesReceived)); - ActivityMetrics.gauge(activityDef, - consumerMetricsPrefix + "recvdMsgsRate", - consumerSafeExtractMetric(consumer, ConsumerStats::getRateMsgsReceived)); - - } catch (PulsarClientException ple) { - ple.printStackTrace(); - throw new RuntimeException("Unable to create a Pulsar consumer!"); - } - - consumers.put(consumerCacheKey, consumer); - } - - return consumer; - } - // - ////////////////////////////////////// - // Multi-topic Consumer Processing <-- end - ////////////////////////////////////// - - - ////////////////////////////////////// - // Reader Processing --> Start - ////////////////////////////////////// - private String getEffectiveReaderTopicName(String cycleReaderTopicName) { - if (!StringUtils.isBlank(cycleReaderTopicName)) { - return cycleReaderTopicName; - } - - String globalReaderTopicName = pulsarNBClientConf.getReaderTopicName(); - if (!StringUtils.isBlank(globalReaderTopicName)) { - return globalReaderTopicName; - } - - throw new RuntimeException("Reader:: Reader topic name must be set at either global level or cycle level!"); - } - - private String getEffectiveReaderName(String cycleReaderName) { - if (!StringUtils.isBlank(cycleReaderName)) { - return cycleReaderName; - } - - String globalReaderName = pulsarNBClientConf.getConsumerName(); - if (!StringUtils.isBlank(globalReaderName)) { - return globalReaderName; - } - - return ""; - } - - private String getEffectiveStartMsgPosStr(String cycleStartMsgPosStr) { - if (!StringUtils.isBlank(cycleStartMsgPosStr)) { - return cycleStartMsgPosStr; - } - - String globalStartMsgPosStr = pulsarNBClientConf.getStartMsgPosStr(); - if (!StringUtils.isBlank(globalStartMsgPosStr)) { - return globalStartMsgPosStr; - } - - return PulsarActivityUtil.READER_MSG_POSITION_TYPE.latest.label; - } - - public Reader getReader(String cycleTopicName, - String cycleReaderName, - String cycleStartMsgPos) { - - String topicName = getEffectiveReaderTopicName(cycleTopicName); - String readerName = getEffectiveReaderName(cycleReaderName); - String startMsgPosStr = getEffectiveStartMsgPosStr(cycleStartMsgPos); - if (!PulsarActivityUtil.isValideReaderStartPosition(startMsgPosStr)) { - throw new RuntimeException("Reader:: Invalid value for reader start message position!"); - } - - String readerCacheKey = PulsarActivityUtil.buildCacheKey(topicName, readerName, startMsgPosStr); - Reader reader = readers.get(readerCacheKey); - - if (reader == null) { - PulsarClient pulsarClient = getPulsarClient(); - - Map readerConf = pulsarNBClientConf.getReaderConfMap(); - - // Remove global level settings: "topicName" and "readerName" - readerConf.remove(PulsarActivityUtil.READER_CONF_STD_KEY.topicName.label); - readerConf.remove(PulsarActivityUtil.READER_CONF_STD_KEY.readerName.label); - // Remove non-standard reader configuration properties - readerConf.remove(PulsarActivityUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label); - - try { - ReaderBuilder readerBuilder = pulsarClient. - newReader(pulsarSchema). - loadConf(readerConf). - topic(topicName). - readerName(readerName); - - MessageId startMsgId = MessageId.latest; - if (startMsgPosStr.equalsIgnoreCase(PulsarActivityUtil.READER_MSG_POSITION_TYPE.earliest.label)) { - startMsgId = MessageId.earliest; - } - //TODO: custom start message position is NOT supported yet - //else if (startMsgPosStr.startsWith(PulsarActivityUtil.READER_MSG_POSITION_TYPE.custom.label)) { - // startMsgId = MessageId.latest; - //} - - reader = readerBuilder.startMessageId(startMsgId).create(); - - } catch (PulsarClientException ple) { - ple.printStackTrace(); - throw new RuntimeException("Unable to create a Pulsar reader!"); - } - - readers.put(readerCacheKey, reader); - } - - return reader; - } - ////////////////////////////////////// - // Reader Processing <-- end - ////////////////////////////////////// -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpaceCache.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpaceCache.java deleted file mode 100644 index b99eaa741..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/PulsarSpaceCache.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar; - -import java.util.concurrent.ConcurrentHashMap; - -/** - * To enable flexibility in testing methods, each object graph which is used within - * the pulsar API is kept within a single umbrella called the PulsarSpace. - * This allows for clients, producers, and consumers to remain connected and - * cached in a useful way. - */ -public class PulsarSpaceCache { - - // TODO: Implement cache limits - // TODO: Implement variant cache eviction behaviors (halt, warn, LRU) - - private final PulsarActivity activity; - private final ConcurrentHashMap clientScopes = new ConcurrentHashMap<>(); - - public PulsarSpaceCache(PulsarActivity pulsarActivity) { - this.activity = pulsarActivity; - } - - public Iterable getAssociatedPulsarSpace() { - return clientScopes.values(); - } - - public PulsarActivity getAssociatedPulsarActivity() { - return activity; - } - - public PulsarSpace getPulsarSpace(String name) { - return clientScopes.computeIfAbsent(name, spaceName -> new PulsarSpace(spaceName, activity)); - } - - public PulsarActivity getActivity() { return activity; } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverParamException.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverParamException.java deleted file mode 100644 index 4fb1895b6..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverParamException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.exception; - -public class PulsarDriverParamException extends RuntimeException { - - public PulsarDriverParamException(String message) { - super(message); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnexpectedException.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnexpectedException.java deleted file mode 100644 index f7ae2da6e..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnexpectedException.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.exception; - -public class PulsarDriverUnexpectedException extends RuntimeException { - - public PulsarDriverUnexpectedException(String message) { - super(message); - } - public PulsarDriverUnexpectedException(Exception e) { super(e); } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnsupportedOpException.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnsupportedOpException.java deleted file mode 100644 index cde63fd0e..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/exception/PulsarDriverUnsupportedOpException.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.exception; - -public class PulsarDriverUnsupportedOpException extends RuntimeException { - - public PulsarDriverUnsupportedOpException() { super("Unsupported Pulsar driver operation type"); } - -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/EndToEndStartingTimeSource.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/EndToEndStartingTimeSource.java deleted file mode 100644 index 2a2fa157a..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/EndToEndStartingTimeSource.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -public enum EndToEndStartingTimeSource { - NONE, // no end-to-end latency calculation - MESSAGE_PUBLISH_TIME, // use message publish timestamp - MESSAGE_EVENT_TIME, // use message event timestamp - MESSAGE_PROPERTY_E2E_STARTING_TIME // use message property called "e2e_starting_time" as the timestamp -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandler.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandler.java deleted file mode 100644 index 59b7b5f6e..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandler.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import java.util.*; -import org.apache.commons.lang3.RandomUtils; - -/** - * Handles adding a monotonic sequence number to message properties of sent messages - */ -class MessageSequenceNumberSendingHandler { - static final int SIMULATED_ERROR_PROBABILITY_PERCENTAGE = 10; - long number = 1; - Queue outOfOrderNumbers; - - public long getNextSequenceNumber(Set simulatedErrorTypes) { - return getNextSequenceNumber(simulatedErrorTypes, SIMULATED_ERROR_PROBABILITY_PERCENTAGE); - } - - long getNextSequenceNumber(Set simulatedErrorTypes, int errorProbabilityPercentage) { - simulateError(simulatedErrorTypes, errorProbabilityPercentage); - return nextNumber(); - } - - private void simulateError(Set simulatedErrorTypes, int errorProbabilityPercentage) { - if (!simulatedErrorTypes.isEmpty() && shouldSimulateError(errorProbabilityPercentage)) { - int selectIndex = 0; - int numberOfErrorTypes = simulatedErrorTypes.size(); - if (numberOfErrorTypes > 1) { - // pick one of the simulated error type randomly - selectIndex = RandomUtils.nextInt(0, numberOfErrorTypes); - } - PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE errorType = simulatedErrorTypes.stream() - .skip(selectIndex) - .findFirst() - .get(); - switch (errorType) { - case OutOfOrder: - // simulate message out of order - injectMessagesOutOfOrder(); - break; - case MsgDup: - // simulate message duplication - injectMessageDuplication(); - break; - case MsgLoss: - // simulate message loss - injectMessageLoss(); - break; - } - } - } - - private boolean shouldSimulateError(int errorProbabilityPercentage) { - // Simulate error with the specified probability - return RandomUtils.nextInt(0, 100) < errorProbabilityPercentage; - } - - long nextNumber() { - if (outOfOrderNumbers != null) { - long nextNumber = outOfOrderNumbers.poll(); - if (outOfOrderNumbers.isEmpty()) { - outOfOrderNumbers = null; - } - return nextNumber; - } - return number++; - } - - void injectMessagesOutOfOrder() { - if (outOfOrderNumbers == null) { - outOfOrderNumbers = new ArrayDeque<>(Arrays.asList(number + 2, number, number + 1)); - number += 3; - } - } - - void injectMessageDuplication() { - if (outOfOrderNumbers == null) { - number--; - } - } - - void injectMessageLoss() { - if (outOfOrderNumbers == null) { - number++; - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminMapper.java deleted file mode 100644 index c3ecd41de..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminMapper.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; - -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The result contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public abstract class PulsarAdminMapper extends PulsarOpMapper { - protected final LongFunction adminDelOpFunc; - - protected PulsarAdminMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc) { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc); - this.adminDelOpFunc = adminDelOpFunc; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceMapper.java deleted file mode 100644 index 1e2943eb2..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceMapper.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; - -import java.util.Set; -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class PulsarAdminNamespaceMapper extends PulsarAdminMapper { - private final LongFunction namespaceFunc; - - public PulsarAdminNamespaceMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc, - LongFunction namespaceFunc) - { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc); - this.namespaceFunc = namespaceFunc; - } - - @Override - public PulsarOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - boolean adminDelOp = adminDelOpFunc.apply(value); - String namespace = namespaceFunc.apply(value); - - return new PulsarAdminNamespaceOp( - clientSpace, - asyncApi, - adminDelOp, - namespace); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceOp.java deleted file mode 100644 index 2c70d9cc8..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminNamespaceOp.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarSpace; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; - -import java.util.concurrent.CompletableFuture; - -public class PulsarAdminNamespaceOp extends PulsarAdminOp { - - private final static Logger logger = LogManager.getLogger(PulsarAdminNamespaceOp.class); - - private final String fullNsName; - - public PulsarAdminNamespaceOp(PulsarSpace clientSpace, - boolean asyncApi, - boolean adminDelOp, - String fullNsName) - { - super(clientSpace, asyncApi, adminDelOp); - this.fullNsName = fullNsName; - } - - @Override - public void run() { - // Do nothing if the namespace name is empty - if ( StringUtils.isBlank(fullNsName) ) return; - - PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin(); - Namespaces namespaces = pulsarAdmin.namespaces(); - - // Admin API - create tenants and namespaces - if (!adminDelOp) { - try { - if (!asyncApi) { - namespaces.createNamespace(fullNsName); - logger.trace("Successfully created namespace \"" + fullNsName + "\" synchronously!"); - } else { - CompletableFuture future = namespaces.createNamespaceAsync(fullNsName); - future.whenComplete((unused, throwable) -> - logger.trace("Successfully created namespace \"" + fullNsName + "\" asynchronously!")) - .exceptionally(ex -> { - logger.error("Failed to create namespace \"" + fullNsName + "\" asynchronously!:" + ex.getMessage()); - return null; - }); - } - } - catch (PulsarAdminException.ConflictException ce) { - // do nothing if the namespace already exists - } - catch (PulsarAdminException e) { - e.printStackTrace(); - throw new RuntimeException("Unexpected error when creating pulsar namespace: " + fullNsName); - } - } - // Admin API - delete tenants and namespaces - else { - try { - if (!asyncApi) { - namespaces.deleteNamespace(fullNsName, true); - logger.trace("Successfully deleted namespace \"" + fullNsName + "\" synchronously!"); - } else { - CompletableFuture future = namespaces.deleteNamespaceAsync(fullNsName, true); - future.whenComplete((unused, throwable) -> - logger.trace("Successfully deleted namespace \"" + fullNsName + "\" asynchronously!")) - .exceptionally(ex -> { - logger.error("Failed to delete namespace \"" + fullNsName + "\" asynchronously!"); - return null; - }); - } - } - catch (PulsarAdminException.NotFoundException nfe) { - // do nothing if the namespace doesn't exist - } - catch (PulsarAdminException e) { - e.printStackTrace(); - throw new RuntimeException("Unexpected error when deleting pulsar namespace: " + fullNsName); - } - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminOp.java deleted file mode 100644 index 46754f7c5..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminOp.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarSpace; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public abstract class PulsarAdminOp extends SyncPulsarOp { - - private final static Logger logger = LogManager.getLogger(PulsarAdminOp.class); - - protected final PulsarSpace clientSpace; - protected final boolean asyncApi; - protected final boolean adminDelOp; - - protected PulsarAdminOp(PulsarSpace clientSpace, - boolean asyncApi, - boolean adminDelOp) - { - this.clientSpace = clientSpace; - this.asyncApi = asyncApi; - this.adminDelOp = adminDelOp; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantMapper.java deleted file mode 100644 index 6bf310f94..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantMapper.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; - -import java.util.Set; -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class PulsarAdminTenantMapper extends PulsarAdminMapper { - private final LongFunction> adminRolesFunc; - private final LongFunction> allowedClustersFunc; - private final LongFunction tenantFunc; - - public PulsarAdminTenantMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc, - LongFunction> adminRolesFunc, - LongFunction> allowedClustersFunc, - LongFunction tenantFunc) - { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc); - this.adminRolesFunc = adminRolesFunc; - this.allowedClustersFunc = allowedClustersFunc; - this.tenantFunc = tenantFunc; - } - - @Override - public PulsarOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - boolean adminDelOp = adminDelOpFunc.apply(value); - Set adminRoleSet = adminRolesFunc.apply(value); - Set allowedClusterSet = allowedClustersFunc.apply(value); - String tenant = tenantFunc.apply(value); - - return new PulsarAdminTenantOp( - clientSpace, - asyncApi, - adminDelOp, - adminRoleSet, - allowedClusterSet, - tenant); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantOp.java deleted file mode 100644 index 149cdd6fe..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTenantOp.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarSpace; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.admin.*; -import org.apache.pulsar.common.policies.data.TenantInfo; - -import java.util.Set; -import java.util.concurrent.CompletableFuture; - -public class PulsarAdminTenantOp extends PulsarAdminOp { - - private final static Logger logger = LogManager.getLogger(PulsarAdminTenantOp.class); - - private final Set adminRoleSet; - private final Set allowedClusterSet; - private final String tenant; - - public PulsarAdminTenantOp(PulsarSpace clientSpace, - boolean asyncApi, - boolean adminDelOp, - Set adminRoleSet, - Set allowedClusterSet, - String tenant) - { - super(clientSpace, asyncApi, adminDelOp); - this.adminRoleSet = adminRoleSet; - this.allowedClusterSet = allowedClusterSet; - this.tenant = tenant; - } - - @Override - public void run() { - // Do nothing if the tenant name is empty - if ( StringUtils.isBlank(tenant) ) return; - - PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin(); - Tenants tenants = pulsarAdmin.tenants(); - Namespaces namespaces = pulsarAdmin.namespaces(); - - // Admin API - create tenants and namespaces - if (!adminDelOp) { - TenantInfo tenantInfo = TenantInfo.builder() - .adminRoles(adminRoleSet) - .allowedClusters(!allowedClusterSet.isEmpty() ? allowedClusterSet : clientSpace.getPulsarClusterMetadata()) - .build(); - - try { - if (!asyncApi) { - tenants.createTenant(tenant, tenantInfo); - if (logger.isDebugEnabled()) { - logger.debug("Successful sync creation of tenant {}", tenant); - } - } else { - CompletableFuture future = tenants.createTenantAsync(tenant, tenantInfo); - future.whenComplete((unused, throwable) -> { - if (logger.isDebugEnabled()) { - logger.debug("Successful async creation of tenant {}", tenant); - } - }).exceptionally(ex -> { - logger.error("Failed async creation of tenant {}", tenant); - return null; - }); - } - } - catch (PulsarAdminException.ConflictException ce) { - // do nothing if the tenant already exists - } - catch (PulsarAdminException e) { - e.printStackTrace(); - throw new RuntimeException("Unexpected error when creating pulsar tenant: " + tenant); - } - } - // Admin API - delete tenants and namespaces - else { - try { - int nsNum = namespaces.getNamespaces(tenant).size(); - - // Only delete a tenant when there is no underlying namespaces - if ( nsNum == 0 ) { - if (!asyncApi) { - tenants.deleteTenant(tenant); - if (logger.isDebugEnabled()) { - logger.debug("Successful sync deletion of tenant {}", tenant); - } - } else { - CompletableFuture future = tenants.deleteTenantAsync(tenant); - future.whenComplete((unused, throwable) -> { - if (logger.isDebugEnabled()) { - logger.debug("Successful async deletion of tenant {}", tenant); - } - }).exceptionally(ex -> { - if (logger.isDebugEnabled()) { - logger.error("Failed async deletion of tenant {}", tenant); - } - return null; - }); - } - } - } - catch (PulsarAdminException.NotFoundException nfe) { - // do nothing if the tenant doesn't exist - } - catch (PulsarAdminException e) { - e.printStackTrace(); - throw new RuntimeException("Unexpected error when deleting pulsar tenant: " + tenant); - } - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicMapper.java deleted file mode 100644 index 647680ffd..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicMapper.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.function.LongFunction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class PulsarAdminTopicMapper extends PulsarAdminMapper { - private final LongFunction topicUriFunc; - private final LongFunction enablePartionFunc; - private final LongFunction partitionNumFunc; - - public PulsarAdminTopicMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc, - LongFunction topicUriFunc, - LongFunction enablePartionFunc, - LongFunction partitionNumFunc) - { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, adminDelOpFunc); - this.topicUriFunc = topicUriFunc; - this.enablePartionFunc = enablePartionFunc; - this.partitionNumFunc = partitionNumFunc; - } - - @Override - public PulsarOp apply(long value) { - String topicUri = topicUriFunc.apply(value); - String enablePartitionStr = enablePartionFunc.apply(value); - String partitionNumStr = partitionNumFunc.apply(value); - boolean asyncApi = asyncApiFunc.apply(value); - boolean adminDelOp = adminDelOpFunc.apply(value); - - if ( StringUtils.isBlank(topicUri) ) { - throw new RuntimeException("\"topic_uri\" parameter can't be empty when creating a Pulsar topic!"); - } - - boolean partitionTopic = BooleanUtils.toBoolean(enablePartitionStr); - - boolean invalidPartStr; - int partitionNum = 0; - if ( StringUtils.isBlank(partitionNumStr) || !StringUtils.isNumeric(partitionNumStr) ) { - invalidPartStr = true; - } else { - partitionNum = Integer.parseInt(partitionNumStr); - invalidPartStr = (partitionNum <= 0); - } - if (partitionTopic && invalidPartStr) { - throw new RuntimeException("Invalid specified value for \"partition_num\" parameter when creating partitioned topic!"); - } - - return new PulsarAdminTopicOp( - clientSpace, - topicUri, - partitionTopic, - partitionNum, - asyncApi, - adminDelOp); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicOp.java deleted file mode 100644 index a5384c715..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarAdminTopicOp.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Topics; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; - - -public class PulsarAdminTopicOp extends PulsarAdminOp { - - private final static Logger logger = LogManager.getLogger(PulsarAdminTopicOp.class); - - private final String topicUri; - private final boolean partitionTopic; - private final int partitionNum; - private final String fullNsName; - - public PulsarAdminTopicOp(PulsarSpace clientSpace, - String topicUri, - boolean partitionTopic, - int partitionNum, - boolean asyncApi, - boolean adminDelOp) - { - super(clientSpace, asyncApi, adminDelOp); - this.topicUri = topicUri; - this.partitionTopic = partitionTopic; - this.partitionNum = partitionNum; - this.fullNsName = PulsarActivityUtil.getFullNamespaceName(this.topicUri); - } - - // Check whether the specified topic already exists - private boolean checkTopicExistence(Topics topics, String topicUri) { - // Check the existence of the topic - List topicListWorkingArea = new ArrayList<>(); - try { - if (!partitionTopic) { - topicListWorkingArea = topics.getList(fullNsName); - } - else { - topicListWorkingArea = topics.getPartitionedTopicList(fullNsName); - } - } - catch (PulsarAdminException.NotFoundException nfe) { - // do nothing - } - catch (PulsarAdminException e) { - e.printStackTrace(); - throw new RuntimeException("Failed to retrieve topic info.for pulsar namespace: " + fullNsName); - } - - return ( !topicListWorkingArea.isEmpty() && topicListWorkingArea.contains(topicUri) ); - } - - @Override - public void run() { - PulsarAdmin pulsarAdmin = clientSpace.getPulsarAdmin(); - Topics topics = pulsarAdmin.topics(); - - try { - // Create the topic - if (!adminDelOp) { - if (!partitionTopic) { - if (!asyncApi) { - topics.createNonPartitionedTopic(topicUri); - logger.trace("Successfully created non-partitioned topic \"" + topicUri + "\" synchronously!"); - } else { - CompletableFuture future = topics.createNonPartitionedTopicAsync(topicUri); - future.whenComplete((unused, throwable) - -> logger.trace("Successfully created non-partitioned topic \"" + topicUri + "\" asynchronously!")) - .exceptionally(ex -> { - logger.error("Failed to create non-partitioned topic \"" + topicUri + "\" asynchronously!"); - return null; - }); - } - } else { - if (!asyncApi) { - topics.createPartitionedTopic(topicUri, partitionNum); - logger.trace("Successfully created partitioned topic \"" + topicUri + "\"" + - "(partition_num: " + partitionNum + ") synchronously!"); - } else { - CompletableFuture future = topics.createPartitionedTopicAsync(topicUri, partitionNum); - future.whenComplete((unused, throwable) - -> logger.trace("Successfully created partitioned topic \"" + topicUri + "\"" + - "(partition_num: " + partitionNum + ") asynchronously!")) - .exceptionally(ex -> { - logger.error("Failed to create partitioned topic \"" + topicUri + "\"" + - "(partition_num: " + partitionNum + ") asynchronously!"); - return null; - }); - } - } - } - // Delete the topic - else { - if (!partitionTopic) { - if (!asyncApi) { - topics.delete(topicUri, true); - logger.trace("Successfully deleted non-partitioned topic \"" + topicUri + "\" synchronously!"); - } else { - CompletableFuture future = topics.deleteAsync(topicUri, true); - future.whenComplete((unused, throwable) - -> logger.trace("Successfully deleted non-partitioned topic \"" + topicUri + "\" asynchronously!")) - .exceptionally(ex -> { - logger.error("Failed to delete non-partitioned topic \"" + topicUri + "\" asynchronously!"); - return null; - }); - } - } else { - if (!asyncApi) { - topics.deletePartitionedTopic(topicUri, true); - logger.trace("Successfully deleted partitioned topic \"" + topicUri + "\" synchronously!"); - } else { - CompletableFuture future = topics.deletePartitionedTopicAsync(topicUri, true); - future.whenComplete((unused, throwable) - -> logger.trace("Successfully deleted partitioned topic \"" + topicUri + "\" asynchronously!")) - .exceptionally(ex -> { - logger.error("Failed to delete partitioned topic \"" + topicUri + "\" asynchronously!"); - return null; - }); - } - } - } - } - catch (PulsarAdminException e) { - e.printStackTrace(); - String errMsg = String.format("Unexpected error when %s pulsar topic: %s (partition topic: %b; partition number: %d)", - (!adminDelOp ? "creating" : "deleting"), - topicUri, - partitionTopic, - partitionNum); - throw new RuntimeException(errMsg); - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndMapper.java deleted file mode 100644 index 6afa68aa3..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndMapper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; - -import java.util.function.LongFunction; - -public class PulsarBatchProducerEndMapper extends PulsarOpMapper { - - public PulsarBatchProducerEndMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc) - { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc); - } - - @Override - public PulsarOp apply(long value) { - return new PulsarBatchProducerEndOp(); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndOp.java deleted file mode 100644 index 9ff6e2b0d..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerEndOp.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.api.errors.BasicError; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.common.util.FutureUtil; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -public class PulsarBatchProducerEndOp extends SyncPulsarOp { - @Override - public void run() { - List> container = PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.get(); - Producer producer = PulsarBatchProducerStartOp.threadLocalProducer.get(); - - if ((container != null) && (!container.isEmpty())) { - try { - // producer.flushAsync().get(); - FutureUtil.waitForAll(container).get(); - } catch (Exception e) { - throw new RuntimeException("Batch Producer:: failed to send (some of) the batched messages!"); - } - - container.clear(); - PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.set(null); - } - else { - throw new BasicError("You tried to end an empty batch message container. This means you" + - " did initiate the batch container properly, or there is an error in your" + - " pulsar op sequencing and ratios."); - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerMapper.java deleted file mode 100644 index 98a0b44ef..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerMapper.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.LongFunction; - -public class PulsarBatchProducerMapper extends PulsarOpMapper { - - private final static Logger logger = LogManager.getLogger(PulsarBatchProducerMapper.class); - - private final LongFunction keyFunc; - private final LongFunction propFunc; - private final LongFunction payloadFunc; - - public PulsarBatchProducerMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction keyFunc, - LongFunction propFunc, - LongFunction payloadFunc) { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc); - this.keyFunc = keyFunc; - this.propFunc = propFunc; - this.payloadFunc = payloadFunc; - } - - @Override - public PulsarOp apply(long value) { - String msgKey = keyFunc.apply(value); - String msgPayload = payloadFunc.apply(value); - - // Check if msgPropJonStr is valid JSON string with a collection of key/value pairs - // - if Yes, convert it to a map - // - otherwise, log an error message and ignore message properties without throwing a runtime exception - Map msgProperties = new HashMap<>(); - String msgPropJsonStr = propFunc.apply(value); - try { - msgProperties = PulsarActivityUtil.convertJsonToMap(msgPropJsonStr); - } - catch (Exception e) { - logger.error( - "PulsarProducerMapper:: Error parsing message property JSON string {}, ignore message properties!", - msgPropJsonStr); - } - - return new PulsarBatchProducerOp( - clientSpace.getPulsarSchema(), - msgKey, - msgProperties, - msgPayload - ); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerOp.java deleted file mode 100644 index 0433d233f..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerOp.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.util.AvroUtil; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import org.apache.pulsar.client.api.MessageId; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.TypedMessageBuilder; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.common.schema.SchemaType; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -public class PulsarBatchProducerOp extends SyncPulsarOp { - - private final Schema pulsarSchema; - private final String msgKey; - private final Map msgProperties; - private final String msgPayload; - - public PulsarBatchProducerOp(Schema schema, - String key, - Map msgProperties, - String payload) { - this.pulsarSchema = schema; - this.msgKey = key; - this.msgProperties = msgProperties; - this.msgPayload = payload; - } - - @Override - public void run() { - if ((msgPayload == null) || msgPayload.isEmpty()) { - throw new RuntimeException("Message payload (\"msg-value\") can't be empty!"); - } - - List> container = PulsarBatchProducerStartOp.threadLocalBatchMsgContainer.get(); - Producer producer = PulsarBatchProducerStartOp.threadLocalProducer.get(); - assert (producer != null) && (container != null); - - TypedMessageBuilder typedMessageBuilder = producer.newMessage(pulsarSchema); - if ((msgKey != null) && (!msgKey.isEmpty())) { - typedMessageBuilder = typedMessageBuilder.key(msgKey); - } - if (!msgProperties.isEmpty()) { - typedMessageBuilder = typedMessageBuilder.properties(msgProperties); - } - - SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); - if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro( - (GenericAvroSchema) pulsarSchema, - pulsarSchema.getSchemaInfo().getSchemaDefinition(), - msgPayload - ); - typedMessageBuilder = typedMessageBuilder.value(payload); - } else { - typedMessageBuilder = typedMessageBuilder.value(msgPayload.getBytes(StandardCharsets.UTF_8)); - } - - container.add(typedMessageBuilder.sendAsync()); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartMapper.java deleted file mode 100644 index 83d80b614..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartMapper.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.pulsar.client.api.Producer; - -import java.util.function.LongFunction; - -public class PulsarBatchProducerStartMapper extends PulsarOpMapper { - - private final LongFunction> batchProducerFunc; - - public PulsarBatchProducerStartMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction> batchProducerFunc) { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc); - this.batchProducerFunc = batchProducerFunc; - } - - @Override - public PulsarOp apply(long value) { - Producer batchProducer = batchProducerFunc.apply(value); - return new PulsarBatchProducerStartOp(batchProducer); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartOp.java deleted file mode 100644 index 65ad3f929..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarBatchProducerStartOp.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.api.errors.BasicError; -import org.apache.commons.compress.utils.Lists; -import org.apache.pulsar.client.api.*; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -public class PulsarBatchProducerStartOp extends SyncPulsarOp { - - // TODO: ensure sane container lifecycle management - public final transient static ThreadLocal>> threadLocalBatchMsgContainer = new ThreadLocal<>(); - public final transient static ThreadLocal> threadLocalProducer = new ThreadLocal<>(); - - public PulsarBatchProducerStartOp(Producer batchProducer) { - threadLocalProducer.set(batchProducer); - } - - @Override - public void run() { - List> container = threadLocalBatchMsgContainer.get(); - - if (container == null) { - container = Lists.newArrayList(); - threadLocalBatchMsgContainer.set(container); - } else { - throw new BasicError("You tried to create a batch message container where one was already" + - " defined. This means you did not flush and unset the last container, or there is an error in your" + - " pulsar op sequencing and ratios."); - } - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java deleted file mode 100644 index 4e5b8c298..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.transaction.Transaction; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.LongFunction; -import java.util.function.Supplier; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class PulsarConsumerMapper extends PulsarTransactOpMapper { - - private final static Logger logger = LogManager.getLogger(PulsarProducerMapper.class); - - private final LongFunction> consumerFunc; - private final EndToEndStartingTimeSource endToEndStartingTimeSource; - private final LongFunction payloadRttFieldFunc; - - public PulsarConsumerMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction useTransactionFunc, - LongFunction seqTrackingFunc, - LongFunction> transactionSupplierFunc, - LongFunction> consumerFunc, - EndToEndStartingTimeSource endToEndStartingTimeSource, - LongFunction payloadRttFieldFunc) { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, useTransactionFunc, seqTrackingFunc, transactionSupplierFunc); - this.consumerFunc = consumerFunc; - this.endToEndStartingTimeSource = endToEndStartingTimeSource; - this.payloadRttFieldFunc = payloadRttFieldFunc; - } - - @Override - public PulsarOp apply(long value) { - boolean seqTracking = seqTrackingFunc.apply(value); - Consumer consumer = consumerFunc.apply(value); - boolean asyncApi = asyncApiFunc.apply(value); - boolean useTransaction = useTransactionFunc.apply(value); - Supplier transactionSupplier = transactionSupplierFunc.apply(value); - String payloadRttFieldFunc = this.payloadRttFieldFunc.apply(value); - - return new PulsarConsumerOp( - pulsarActivity, - asyncApi, - useTransaction, - seqTracking, - transactionSupplier, - consumer, - clientSpace.getPulsarSchema(), - clientSpace.getPulsarClientConf().getConsumerTimeoutSeconds(), - endToEndStartingTimeSource, - this::getReceivedMessageSequenceTracker, - payloadRttFieldFunc); - } - - - private ReceivedMessageSequenceTracker getReceivedMessageSequenceTracker(String topicName) { - return receivedMessageSequenceTrackersForTopicThreadLocal.get() - .computeIfAbsent(topicName, k -> createReceivedMessageSequenceTracker()); - } - - private ReceivedMessageSequenceTracker createReceivedMessageSequenceTracker() { - return new ReceivedMessageSequenceTracker(pulsarActivity.getMsgErrOutOfSeqCounter(), - pulsarActivity.getMsgErrDuplicateCounter(), - pulsarActivity.getMsgErrLossCounter()); - } - - private final ThreadLocal> receivedMessageSequenceTrackersForTopicThreadLocal = - ThreadLocal.withInitial(HashMap::new); - -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java deleted file mode 100644 index 00ae946c6..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Function; -import java.util.function.Supplier; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import org.apache.commons.lang3.StringUtils; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnexpectedException; -import io.nosqlbench.driver.pulsar.util.AvroUtil; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.api.*; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.transaction.Transaction; -import org.apache.pulsar.common.schema.SchemaType; - -public class PulsarConsumerOp implements PulsarOp { - - private final static Logger logger = LogManager.getLogger(PulsarConsumerOp.class); - - private final PulsarActivity pulsarActivity; - - private final boolean asyncPulsarOp; - private final boolean useTransaction; - private final boolean seqTracking; - private final Supplier transactionSupplier; - - private final Consumer consumer; - private final Schema pulsarSchema; - private final int timeoutSeconds; - private final EndToEndStartingTimeSource endToEndStartingTimeSource; - - private final Counter bytesCounter; - private final Histogram messageSizeHistogram; - private final Timer transactionCommitTimer; - - // keep track of end-to-end message latency - private final Histogram e2eMsgProcLatencyHistogram; - - private final Function receivedMessageSequenceTrackerForTopic; - private final Histogram payloadRttHistogram; - private final String payloadRttTrackingField; - - private org.apache.avro.Schema avroSchema; - - public PulsarConsumerOp( - PulsarActivity pulsarActivity, - boolean asyncPulsarOp, - boolean useTransaction, - boolean seqTracking, - Supplier transactionSupplier, - Consumer consumer, - Schema schema, - int timeoutSeconds, - EndToEndStartingTimeSource endToEndStartingTimeSource, - Function receivedMessageSequenceTrackerForTopic, - String payloadRttTrackingField) - { - this.pulsarActivity = pulsarActivity; - - this.asyncPulsarOp = asyncPulsarOp; - this.useTransaction = useTransaction; - this.seqTracking = seqTracking; - this.transactionSupplier = transactionSupplier; - - this.consumer = consumer; - this.pulsarSchema = schema; - this.timeoutSeconds = timeoutSeconds; - this.endToEndStartingTimeSource = endToEndStartingTimeSource; - - this.bytesCounter = pulsarActivity.getBytesCounter(); - this.messageSizeHistogram = pulsarActivity.getMessageSizeHistogram(); - this.transactionCommitTimer = pulsarActivity.getCommitTransactionTimer(); - - this.e2eMsgProcLatencyHistogram = pulsarActivity.getE2eMsgProcLatencyHistogram(); - this.payloadRttHistogram = pulsarActivity.getPayloadRttHistogram(); - this.receivedMessageSequenceTrackerForTopic = receivedMessageSequenceTrackerForTopic; - this.payloadRttTrackingField = payloadRttTrackingField; - } - - private void checkAndUpdateMessageErrorCounter(Message message) { - String msgSeqIdStr = message.getProperty(PulsarActivityUtil.MSG_SEQUENCE_NUMBER); - - if ( !StringUtils.isBlank(msgSeqIdStr) ) { - long sequenceNumber = Long.parseLong(msgSeqIdStr); - ReceivedMessageSequenceTracker receivedMessageSequenceTracker = receivedMessageSequenceTrackerForTopic.apply(message.getTopicName()); - receivedMessageSequenceTracker.sequenceNumberReceived(sequenceNumber); - } - } - - @Override - public void run(Runnable timeTracker) { - - final Transaction transaction; - if (useTransaction) { - // if you are in a transaction you cannot set the schema per-message - transaction = transactionSupplier.get(); - } - else { - transaction = null; - } - - if (!asyncPulsarOp) { - try { - Message message; - - if (timeoutSeconds <= 0) { - // wait forever - message = consumer.receive(); - } - else { - message = consumer - .receive(timeoutSeconds, TimeUnit.SECONDS); - if (message == null) { - throw new TimeoutException("Did not receive a message within "+timeoutSeconds+" seconds"); - } - } - - handleMessage(transaction, message); - } - catch (Exception e) { - logger.error( - "Sync message receiving failed - timeout value: {} seconds ", timeoutSeconds, e); - throw new PulsarDriverUnexpectedException("" + - "Sync message receiving failed - timeout value: " + timeoutSeconds + " seconds "); - } - } - else { - try { - CompletableFuture> msgRecvFuture = consumer.receiveAsync(); - if (useTransaction) { - // add commit step - msgRecvFuture = msgRecvFuture.thenCompose(msg -> { - Timer.Context ctx = transactionCommitTimer.time(); - return transaction - .commit() - .whenComplete((m,e) -> ctx.close()) - .thenApply(v-> msg); - } - ); - } - - msgRecvFuture.thenAccept(message -> { - try { - handleMessage(transaction, message); - } catch (PulsarClientException | TimeoutException e) { - pulsarActivity.asyncOperationFailed(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - pulsarActivity.asyncOperationFailed(e.getCause()); - } - }).exceptionally(ex -> { - pulsarActivity.asyncOperationFailed(ex); - return null; - }); - } - catch (Exception e) { - throw new PulsarDriverUnexpectedException(e); - } - } - } - - private void handleMessage(Transaction transaction, Message message) - throws PulsarClientException, InterruptedException, ExecutionException, TimeoutException { - - // acknowledge the message as soon as possible - if (!useTransaction) { - consumer.acknowledgeAsync(message.getMessageId()) - .get(timeoutSeconds, TimeUnit.SECONDS); - } else { - consumer.acknowledgeAsync(message.getMessageId(), transaction) - .get(timeoutSeconds, TimeUnit.SECONDS); - - // little problem: here we are counting the "commit" time - // inside the overall time spent for the execution of the consume operation - // we should refactor this operation as for PulsarProducerOp, and use the passed callback - // to track with precision the time spent for the operation and for the commit - try (Timer.Context ctx = transactionCommitTimer.time()) { - transaction.commit().get(); - } - } - - if (logger.isDebugEnabled()) { - SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); - - if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - org.apache.avro.Schema avroSchema = getSchemaFromConfiguration(); - org.apache.avro.generic.GenericRecord avroGenericRecord = - AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData()); - - logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}", - consumer.getConsumerName(), - message.getKey(), - message.getProperties(), - avroGenericRecord.toString()); - } - else { - logger.debug("({}) message received: msg-key={}; msg-properties={}; msg-payload={}", - consumer.getConsumerName(), - message.getKey(), - message.getProperties(), - new String(message.getData())); - } - } - - if (!payloadRttTrackingField.isEmpty()) { - Object decodedPayload = message.getValue(); - Long extractedSendTime = null; - // if Pulsar is able to decode this it is better to let it do the work - // because Pulsar caches the Schema, handles Schema evolution - // as much efficiently as possible - if (decodedPayload instanceof GenericRecord) { - GenericRecord pulsarGenericRecord = (GenericRecord) decodedPayload; - Object field = pulsarGenericRecord.getField(payloadRttTrackingField); - if (field != null) { - if (field instanceof Number) { - extractedSendTime = ((Number) field).longValue(); - } else { - extractedSendTime = Long.valueOf(field.toString()); - } - } - } else { - org.apache.avro.Schema avroSchema = getSchemaFromConfiguration(); - org.apache.avro.generic.GenericRecord avroGenericRecord = - AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData()); - if (avroGenericRecord.hasField(payloadRttTrackingField)) { - extractedSendTime = (Long) avroGenericRecord.get(payloadRttTrackingField); - } - } - if (extractedSendTime != null) { - long delta = System.currentTimeMillis() - extractedSendTime; - payloadRttHistogram.update(delta); - } - } - - // keep track end-to-end message processing latency - if (endToEndStartingTimeSource != EndToEndStartingTimeSource.NONE) { - long startTimeStamp = 0L; - switch (endToEndStartingTimeSource) { - case MESSAGE_PUBLISH_TIME: - startTimeStamp = message.getPublishTime(); - break; - case MESSAGE_EVENT_TIME: - startTimeStamp = message.getEventTime(); - break; - case MESSAGE_PROPERTY_E2E_STARTING_TIME: - String startingTimeProperty = message.getProperty("e2e_starting_time"); - startTimeStamp = startingTimeProperty != null ? Long.parseLong(startingTimeProperty) : 0L; - break; - } - if (startTimeStamp != 0L) { - long e2eMsgLatency = System.currentTimeMillis() - startTimeStamp; - e2eMsgProcLatencyHistogram.update(e2eMsgLatency); - } - } - - // keep track of message errors and update error counters - if (seqTracking) checkAndUpdateMessageErrorCounter(message); - - int messageSize = message.getData().length; - bytesCounter.inc(messageSize); - messageSizeHistogram.update(messageSize); - } - - private org.apache.avro.Schema getSchemaFromConfiguration() { - String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition(); - // no need for synchronization, this is only a cache - // in case of the race we will parse the string twice, not a big - if (avroSchema == null) { - avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr); - } - return avroSchema; - } - -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOp.java deleted file mode 100644 index abc936278..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOp.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -/** - * Base type of all Pulsar Operations including Producers and Consumers. - */ -public interface PulsarOp { - - /** - * Execute the operation, invoke the timeTracker when the operation ended. - * The timeTracker can be invoked in a separate thread, it is only used for metrics. - * @param timeTracker - */ - void run(Runnable timeTracker); -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOpMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOpMapper.java deleted file mode 100644 index 80d920acb..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarOpMapper.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.transaction.Transaction; - -import java.util.function.LongFunction; -import java.util.function.Supplier; - -public abstract class PulsarOpMapper implements LongFunction { - protected final CommandTemplate cmdTpl; - protected final PulsarSpace clientSpace; - protected final PulsarActivity pulsarActivity; - protected final LongFunction asyncApiFunc; - - public PulsarOpMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc) - { - this.cmdTpl = cmdTpl; - this.clientSpace = clientSpace; - this.pulsarActivity = pulsarActivity; - this.asyncApiFunc = asyncApiFunc; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerMapper.java deleted file mode 100644 index 5a81e5582..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerMapper.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.function.LongFunction; -import java.util.function.Supplier; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.transaction.Transaction; - -/** - * This maps a set of specifier functions to a pulsar operation. The pulsar operation contains - * enough state to define a pulsar operation such that it can be executed, measured, and possibly - * retried if needed. - * - * This function doesn't act *as* the operation. It merely maps the construction logic into - * a simple functional type, given the component functions. - * - * For additional parameterization, the command template is also provided. - */ -public class PulsarProducerMapper extends PulsarTransactOpMapper { - - private final static Logger logger = LogManager.getLogger(PulsarProducerMapper.class); - - private final LongFunction> producerFunc; - private final Set seqErrSimuTypes; - private final LongFunction keyFunc; - private final LongFunction propFunc; - private final LongFunction payloadFunc; - - public PulsarProducerMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction useTransactionFunc, - LongFunction seqTrackingFunc, - LongFunction> transactionSupplierFunc, - LongFunction> producerFunc, - Set seqErrSimuTypes, - LongFunction keyFunc, - LongFunction propFunc, - LongFunction payloadFunc) { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc, useTransactionFunc, seqTrackingFunc, transactionSupplierFunc); - - this.producerFunc = producerFunc; - this.seqErrSimuTypes = seqErrSimuTypes; - this.keyFunc = keyFunc; - this.propFunc = propFunc; - this.payloadFunc = payloadFunc; - } - - @Override - public PulsarOp apply(long value) { - boolean asyncApi = asyncApiFunc.apply(value); - boolean useTransaction = useTransactionFunc.apply(value); - Supplier transactionSupplier = transactionSupplierFunc.apply(value); - - Producer producer = producerFunc.apply(value); - - String msgKey = keyFunc.apply(value); - String msgPayload = payloadFunc.apply(value); - - // Check if msgPropJonStr is valid JSON string with a collection of key/value pairs - // - if Yes, convert it to a map - // - otherwise, log an error message and ignore message properties without throwing a runtime exception - Map msgProperties = new HashMap<>(); - String msgPropJsonStr = propFunc.apply(value); - if (!StringUtils.isBlank(msgPropJsonStr)) { - try { - msgProperties = PulsarActivityUtil.convertJsonToMap(msgPropJsonStr); - - } catch (Exception e) { - logger.error( - "Error parsing message property JSON string {}, ignore message properties!", - msgPropJsonStr); - } - } - - boolean sequenceTrackingEnabled = seqTrackingFunc.apply(value); - if (sequenceTrackingEnabled) { - long nextSequenceNumber = getMessageSequenceNumberSendingHandler(producer.getTopic()) - .getNextSequenceNumber(seqErrSimuTypes); - msgProperties.put(PulsarActivityUtil.MSG_SEQUENCE_NUMBER, String.valueOf(nextSequenceNumber)); - } - - return new PulsarProducerOp( - pulsarActivity, - asyncApi, - useTransaction, - transactionSupplier, - producer, - clientSpace.getPulsarSchema(), - msgKey, - msgProperties, - msgPayload); - } - - private MessageSequenceNumberSendingHandler getMessageSequenceNumberSendingHandler(String topicName) { - return MessageSequenceNumberSendingHandlersThreadLocal.get() - .computeIfAbsent(topicName, k -> new MessageSequenceNumberSendingHandler()); - } - - private final ThreadLocal> MessageSequenceNumberSendingHandlersThreadLocal = - ThreadLocal.withInitial(HashMap::new); - -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerOp.java deleted file mode 100644 index 611da9e19..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarProducerOp.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Timer; -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException; -import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnexpectedException; -import io.nosqlbench.driver.pulsar.util.AvroUtil; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.api.*; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.KeyValueSchema; -import org.apache.pulsar.client.api.transaction.Transaction; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.SchemaType; - -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.function.Supplier; - -public class PulsarProducerOp implements PulsarOp { - - private final static Logger logger = LogManager.getLogger(PulsarProducerOp.class); - - private final PulsarActivity pulsarActivity; - - private final boolean asyncPulsarOp; - private final boolean useTransaction; - private final Supplier transactionSupplier; - - private final Producer producer; - private final Schema pulsarSchema; - private final String msgKey; - private final Map msgProperties; - private final String msgPayload; - - private final Counter bytesCounter; - private final Histogram messageSizeHistogram; - private final Timer transactionCommitTimer; - - private org.apache.avro.Schema avroSchema; - private org.apache.avro.Schema avroKeySchema; - - public PulsarProducerOp( PulsarActivity pulsarActivity, - boolean asyncPulsarOp, - boolean useTransaction, - Supplier transactionSupplier, - Producer producer, - Schema schema, - String key, - Map msgProperties, - String payload) { - this.pulsarActivity = pulsarActivity; - - this.asyncPulsarOp = asyncPulsarOp; - this.useTransaction = useTransaction; - this.transactionSupplier = transactionSupplier; - - this.producer = producer; - this.pulsarSchema = schema; - this.msgKey = key; - this.msgProperties = msgProperties; - this.msgPayload = payload; - - this.bytesCounter = pulsarActivity.getBytesCounter(); - this.messageSizeHistogram = pulsarActivity.getMessageSizeHistogram(); - this.transactionCommitTimer = pulsarActivity.getCommitTransactionTimer(); - } - - @Override - public void run(Runnable timeTracker) { - if ( StringUtils.isBlank(msgPayload)) { - throw new PulsarDriverParamException("Message payload (\"msg-value\") can't be empty!"); - } - - TypedMessageBuilder typedMessageBuilder; - - final Transaction transaction; - if (useTransaction) { - // if you are in a transaction you cannot set the schema per-message - transaction = transactionSupplier.get(); - typedMessageBuilder = producer.newMessage(transaction); - } - else { - transaction = null; - typedMessageBuilder = producer.newMessage(pulsarSchema); - } - - // set message key - if (!StringUtils.isBlank(msgKey)) { - typedMessageBuilder = typedMessageBuilder.key(msgKey); - } - - // set message properties - if ( !msgProperties.isEmpty() ) { - typedMessageBuilder = typedMessageBuilder.properties(msgProperties); - } - - // set message payload - int messageSize; - SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); - if (pulsarSchema instanceof KeyValueSchema) { - - // {KEY IN JSON}||{VALUE IN JSON} - int separator = msgPayload.indexOf("}||{"); - if (separator < 0) { - throw new IllegalArgumentException("KeyValue payload MUST be in form {KEY IN JSON}||{VALUE IN JSON} (with 2 pipes that separate the KEY part from the VALUE part)"); - } - String keyInput = msgPayload.substring(0, separator + 1); - String valueInput = msgPayload.substring(separator + 3); - - KeyValueSchema keyValueSchema = (KeyValueSchema) pulsarSchema; - org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); - GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro( - (GenericAvroSchema) keyValueSchema.getValueSchema(), - avroSchema, - valueInput - ); - - org.apache.avro.Schema avroSchemaForKey = getKeyAvroSchemaFromConfiguration(); - GenericRecord key = AvroUtil.GetGenericRecord_PulsarAvro( - (GenericAvroSchema) keyValueSchema.getKeySchema(), - avroSchemaForKey, - keyInput - ); - typedMessageBuilder = typedMessageBuilder.value(new KeyValue(key, payload)); - // TODO: add a way to calculate the message size for KEY_VALUE messages - messageSize = msgPayload.length(); - } else if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - GenericRecord payload = AvroUtil.GetGenericRecord_PulsarAvro( - (GenericAvroSchema) pulsarSchema, - pulsarSchema.getSchemaInfo().getSchemaDefinition(), - msgPayload - ); - typedMessageBuilder = typedMessageBuilder.value(payload); - // TODO: add a way to calculate the message size for AVRO messages - messageSize = msgPayload.length(); - } else { - byte[] array = msgPayload.getBytes(StandardCharsets.UTF_8); - typedMessageBuilder = typedMessageBuilder.value(array); - messageSize = array.length; - } - messageSizeHistogram.update(messageSize); - bytesCounter.inc(messageSize); - - //TODO: add error handling with failed message production - if (!asyncPulsarOp) { - try { - logger.trace("Sending message"); - typedMessageBuilder.send(); - - if (useTransaction) { - try (Timer.Context ctx = transactionCommitTimer.time()) { - transaction.commit().get(); - } - } - - if (logger.isDebugEnabled()) { - if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); - org.apache.avro.generic.GenericRecord avroGenericRecord = - AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload); - - logger.debug("({}) Sync message sent: msg-key={}; msg-properties={}; msg-payload={})", - producer.getProducerName(), - msgKey, - msgProperties, - avroGenericRecord.toString()); - } - else { - logger.debug("({}) Sync message sent; msg-key={}; msg-properties={}; msg-payload={}", - producer.getProducerName(), - msgKey, - msgProperties, - msgPayload); - } - } - } - catch (PulsarClientException | ExecutionException | InterruptedException pce) { - String errMsg = - "Sync message sending failed: " + - "key - " + msgKey + "; " + - "properties - " + msgProperties + "; " + - "payload - " + msgPayload; - - logger.trace(errMsg); - - throw new PulsarDriverUnexpectedException(errMsg); - } - - timeTracker.run(); - } - else { - try { - // we rely on blockIfQueueIsFull in order to throttle the request in this case - CompletableFuture future = typedMessageBuilder.sendAsync(); - - if (useTransaction) { - // add commit step - future = future.thenCompose(msg -> { - Timer.Context ctx = transactionCommitTimer.time(); - return transaction - .commit() - .whenComplete((m,e) -> ctx.close()) - .thenApply(v-> msg); - } - ); - } - - future.whenComplete((messageId, error) -> { - if (logger.isDebugEnabled()) { - if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - org.apache.avro.Schema avroSchema = getAvroSchemaFromConfiguration(); - org.apache.avro.generic.GenericRecord avroGenericRecord = - AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, msgPayload); - - logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={})", - producer.getProducerName(), - msgKey, - msgProperties, - avroGenericRecord.toString()); - } - else { - logger.debug("({}) Aysnc message sent: msg-key={}; msg-properties={}; msg-payload={}", - producer.getProducerName(), - msgKey, - msgProperties, - msgPayload); - } - } - - timeTracker.run(); - }).exceptionally(ex -> { - logger.error("Async message sending failed: " + - "key - " + msgKey + "; " + - "properties - " + msgProperties + "; " + - "payload - " + msgPayload); - - pulsarActivity.asyncOperationFailed(ex); - return null; - }); - } - catch (Exception e) { - throw new PulsarDriverUnexpectedException(e); - } - } - } - - private org.apache.avro.Schema getAvroSchemaFromConfiguration() { - // no need for synchronization, this is only a cache - // in case of the race we will parse the string twice, not a big - if (avroSchema == null) { - if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { - KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; - Schema valueSchema = kvSchema.getValueSchema(); - String avroDefStr = valueSchema.getSchemaInfo().getSchemaDefinition(); - avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr); - } else { - String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition(); - avroSchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr); - } - } - return avroSchema; - } - - private org.apache.avro.Schema getKeyAvroSchemaFromConfiguration() { - // no need for synchronization, this is only a cache - // in case of the race we will parse the string twice, not a big - if (avroKeySchema == null) { - if (pulsarSchema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { - KeyValueSchema kvSchema = (KeyValueSchema) pulsarSchema; - Schema keySchema = kvSchema.getKeySchema(); - String avroDefStr = keySchema.getSchemaInfo().getSchemaDefinition(); - avroKeySchema = AvroUtil.GetSchema_ApacheAvro(avroDefStr); - } else { - throw new RuntimeException("We are not using KEY_VALUE schema, so no Schema for the Key!"); - } - } - return avroKeySchema; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderMapper.java deleted file mode 100644 index 59bef3330..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderMapper.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.pulsar.client.api.Reader; - -import java.util.function.LongFunction; - -public class PulsarReaderMapper extends PulsarOpMapper { - - private final LongFunction> readerFunc; - - public PulsarReaderMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction> readerFunc) - { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc); - this.readerFunc = readerFunc; - } - - @Override - public PulsarOp apply(long value) { - Reader reader = readerFunc.apply(value); - boolean asyncApi = asyncApiFunc.apply(value); - - return new PulsarReaderOp( - reader, - clientSpace.getPulsarSchema(), - asyncApi - ); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderOp.java deleted file mode 100644 index 7030f08c6..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarReaderOp.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.util.AvroUtil; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Reader; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.common.schema.SchemaType; - -public class PulsarReaderOp extends SyncPulsarOp { - private final Reader reader; - private final Schema pulsarSchema; - private final boolean asyncPulsarOp; - - public PulsarReaderOp(Reader reader, Schema schema, boolean asyncPulsarOp) { - this.reader = reader; - this.pulsarSchema = schema; - this.asyncPulsarOp = asyncPulsarOp; - } - - public void syncRead() { - try { - SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); - String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition(); - - // TODO: how many messages to read per NB cycle? - Message message; - while (reader.hasMessageAvailable()) { - message = reader.readNext(); - - if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - org.apache.avro.Schema avroSchema = - AvroUtil.GetSchema_ApacheAvro(avroDefStr); - - org.apache.avro.generic.GenericRecord avroGenericRecord = - AvroUtil.GetGenericRecord_ApacheAvro(avroSchema, message.getData()); - - System.out.println("msg-key=" + message.getKey() + " msg-payload=" + avroGenericRecord.toString()); - } else { - System.out.println("msg-key=" + message.getKey() + " msg-payload=" + new String(message.getData())); - } - } - } catch (PulsarClientException e) { - throw new RuntimeException(e); - } - } - - public void asyncRead() { - //TODO: add support for async read - } - - @Override - public void run() { - if (!asyncPulsarOp) - syncRead(); - else - asyncRead(); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarTransactOpMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarTransactOpMapper.java deleted file mode 100644 index 47845ea18..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarTransactOpMapper.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.pulsar.client.api.transaction.Transaction; - -import java.util.function.LongFunction; -import java.util.function.Supplier; - -public abstract class PulsarTransactOpMapper extends PulsarOpMapper { - protected final LongFunction useTransactionFunc; - protected final LongFunction seqTrackingFunc; - protected final LongFunction> transactionSupplierFunc; - - public PulsarTransactOpMapper(CommandTemplate cmdTpl, - PulsarSpace clientSpace, - PulsarActivity pulsarActivity, - LongFunction asyncApiFunc, - LongFunction useTransactionFunc, - LongFunction seqTrackingFunc, - LongFunction> transactionSupplierFunc) - { - super(cmdTpl, clientSpace, pulsarActivity, asyncApiFunc); - this.useTransactionFunc = useTransactionFunc; - this.seqTrackingFunc = seqTrackingFunc; - this.transactionSupplierFunc = transactionSupplierFunc; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReadyPulsarOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReadyPulsarOp.java deleted file mode 100644 index eefdd3f20..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReadyPulsarOp.java +++ /dev/null @@ -1,541 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import io.nosqlbench.driver.pulsar.PulsarActivity; -import io.nosqlbench.driver.pulsar.PulsarSpace; -import io.nosqlbench.driver.pulsar.PulsarSpaceCache; -import io.nosqlbench.driver.pulsar.exception.PulsarDriverParamException; -import io.nosqlbench.driver.pulsar.exception.PulsarDriverUnsupportedOpException; -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; -import io.nosqlbench.engine.api.templating.CommandTemplate; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.Reader; -import org.apache.pulsar.client.api.transaction.Transaction; - -import java.util.*; -import java.util.function.LongFunction; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -public class ReadyPulsarOp extends BaseOpDispenser { - - // TODO: Add this to the pulsar driver docs - public static final String RTT_TRACKING_FIELD = "payload-tracking-field"; - private final static Logger logger = LogManager.getLogger(ReadyPulsarOp.class); - - private final OpTemplate opTpl; - private final CommandTemplate cmdTpl; - private final PulsarSpace clientSpace; - private final LongFunction opFunc; - private final PulsarActivity pulsarActivity; - - // TODO: Add docs for the command template with respect to the OpTemplate - - public ReadyPulsarOp(OpTemplate optpl, PulsarSpaceCache pcache, PulsarActivity pulsarActivity) { - super(optpl); - // TODO: Consider parsing map structures into equivalent binding representation - this.pulsarActivity = pulsarActivity; - this.opTpl = optpl; - this.cmdTpl = new CommandTemplate(optpl); - - // TODO: At the moment, only supports static "client" - String client_name = lookupStaticParameter("client", false, "default"); - this.clientSpace = pcache.getPulsarSpace(client_name); - - this.opFunc = resolve(); - } - - @Override - public PulsarOp apply(long value) { - return opFunc.apply(value); - } - - private LongFunction resolve() { - - String stmtOpType = lookupStaticParameter("optype", true, null); - - if (cmdTpl.containsKey("topic_url")) { - throw new PulsarDriverParamException("[resolve()] \"topic_url\" parameter is not valid. Perhaps you mean \"topic_uri\"?"); - } - - // Doc-level parameter: topic_uri - LongFunction topicUriFunc = lookupParameterFunc(PulsarActivityUtil.DOC_LEVEL_PARAMS.TOPIC_URI.label); - logger.info("topic_uri: {}", topicUriFunc.apply(0)); - - // Doc-level parameter: async_api - boolean useAsyncApi = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ASYNC_API.label, false, "false")); - LongFunction asyncApiFunc = (l) -> useAsyncApi; - logger.info("async_api: {}", useAsyncApi); - - // Doc-level parameter: use_transaction - boolean useTransaction = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.USE_TRANSACTION.label, false, "false")); - LongFunction useTransactionFunc = (l) -> useTransaction; - logger.info("use_transaction: {}", useTransaction); - - // Doc-level parameter: admin_delop - boolean adminDelOp = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.ADMIN_DELOP.label, false, "false")); - LongFunction adminDelOpFunc = (l) -> adminDelOp; - logger.info("admin_delop: {}", adminDelOp); - - // Doc-level parameter: seq_tracking - boolean seqTracking = BooleanUtils.toBoolean(lookupStaticParameter(PulsarActivityUtil.DOC_LEVEL_PARAMS.SEQ_TRACKING.label, false, "false")); - LongFunction seqTrackingFunc = (l) -> seqTracking; - logger.info("seq_tracking: {}", seqTracking); - - // TODO: Collapse this pattern into a simple version and flatten out all call sites - LongFunction payloadRttFieldFunc = lookupParameterFunc(RTT_TRACKING_FIELD, false, ""); - logger.info("payload_rtt_field_func: {}", payloadRttFieldFunc.apply(0)); - - // TODO: Complete implementation for websocket-producer and managed-ledger - // Admin operation: create/delete tenant - if ( StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TENANT.label) ) { - return resolveAdminTenant(clientSpace, asyncApiFunc, adminDelOpFunc); - } - // Admin operation: create/delete namespace - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_NAMESPACE.label)) { - return resolveAdminNamespace(clientSpace, asyncApiFunc, adminDelOpFunc); - } - // Admin operation: create/delete topic - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.ADMIN_TOPIC.label)) { - return resolveAdminTopic(clientSpace, topicUriFunc, asyncApiFunc, adminDelOpFunc); - } - // Regular/non-admin operation: single message sending (producer) - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_SEND.label)) { - return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc); - } - // Regular/non-admin operation: single message consuming from a single topic (consumer) - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_CONSUME.label)) { - return resolveMsgConsume( - clientSpace, - topicUriFunc, - asyncApiFunc, - useTransactionFunc, - seqTrackingFunc, - parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE), - payloadRttFieldFunc); - } - // Regular/non-admin operation: single message consuming from multiple-topics (consumer) - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_MULTI_CONSUME.label)) { - return resolveMultiTopicMsgConsume( - clientSpace, - topicUriFunc, - asyncApiFunc, - useTransactionFunc, - seqTrackingFunc, - payloadRttFieldFunc); - } - // Regular/non-admin operation: single message consuming a single topic (reader) - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.MSG_READ.label)) { - return resolveMsgRead(clientSpace, topicUriFunc, asyncApiFunc); - } -// // Regular/non-admin operation: batch message processing - batch start -// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_START.label)) { -// return resolveMsgBatchSendStart(clientSpace, topicUriFunc, asyncApiFunc); -// } -// // Regular/non-admin operation: batch message processing - message sending (producer) -// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND.label)) { -// return resolveMsgBatchSend(clientSpace, asyncApiFunc); -// } -// // Regular/non-admin operation: batch message processing - batch send -// else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.BATCH_MSG_SEND_END.label)) { -// return resolveMsgBatchSendEnd(clientSpace, asyncApiFunc); -// } - // Regular/non-admin operation: end-to-end message processing - sending message - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_SEND.label)) { - return resolveMsgSend(clientSpace, topicUriFunc, asyncApiFunc, useTransactionFunc, seqTrackingFunc); - } - // Regular/non-admin operation: end-to-end message processing - consuming message - else if (StringUtils.equalsIgnoreCase(stmtOpType, PulsarActivityUtil.OP_TYPES.E2E_MSG_PROC_CONSUME.label)) { - return resolveMsgConsume( - clientSpace, - topicUriFunc, - asyncApiFunc, - useTransactionFunc, - seqTrackingFunc, - parseEndToEndStartingTimeSourceParameter( - EndToEndStartingTimeSource.MESSAGE_PUBLISH_TIME), - payloadRttFieldFunc); - } - // Invalid operation type - else { - throw new PulsarDriverUnsupportedOpException(); - } - } - - private EndToEndStartingTimeSource parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource defaultValue) { - EndToEndStartingTimeSource endToEndStartingTimeSource = defaultValue; - if (cmdTpl.isStatic(PulsarActivityUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label)) { - endToEndStartingTimeSource = EndToEndStartingTimeSource.valueOf(cmdTpl.getStatic(PulsarActivityUtil.DOC_LEVEL_PARAMS.E2E_STARTING_TIME_SOURCE.label).toUpperCase()); - } - return endToEndStartingTimeSource; - } - - // Admin API: create tenant - private LongFunction resolveAdminTenant( - PulsarSpace clientSpace, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc) - { - // "admin_roles" includes comma-separated admin roles: - // e.g. role1, role2 - Set roleSet = lookupStaticParameterSet("admin_roles"); - LongFunction> adminRolesFunc = (l) -> roleSet; - - // "allowed_cluster" includes comma-separated cluster names: - // e.g. cluster1, cluster2 - Set clusterSet = lookupStaticParameterSet("allowed_clusters"); - LongFunction> allowedClustersFunc = (l) -> clusterSet; - - LongFunction tenantFunc = lookupParameterFunc("tenant"); - - return new PulsarAdminTenantMapper( - cmdTpl, - clientSpace, - pulsarActivity, - asyncApiFunc, - adminDelOpFunc, - adminRolesFunc, - allowedClustersFunc, - tenantFunc); - } - - private Set lookupStaticParameterSet(String parameterName) { - return Optional.ofNullable(lookupStaticParameter(parameterName)) - .map(value -> { - Set set = Arrays.stream(value.split(",")) - .map(String::trim) - .collect(Collectors.toCollection(LinkedHashSet::new)); - return set; - }).orElse(Collections.emptySet()); - } - - // Admin API: create tenant - private LongFunction resolveAdminNamespace( - PulsarSpace clientSpace, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc) - { - LongFunction namespaceFunc = lookupParameterFunc("namespace"); - - return new PulsarAdminNamespaceMapper( - cmdTpl, - clientSpace, - pulsarActivity, - asyncApiFunc, - adminDelOpFunc, - namespaceFunc); - } - - // Admin API: create partitioned topic - private LongFunction resolveAdminTopic( - PulsarSpace clientSpace, - LongFunction topic_uri_fun, - LongFunction asyncApiFunc, - LongFunction adminDelOpFunc - ) { - LongFunction enablePartionFunc = lookupParameterFunc("enable_partition"); - - LongFunction partitionNumFunc = lookupParameterFunc("partition_num"); - - return new PulsarAdminTopicMapper( - cmdTpl, - clientSpace, - pulsarActivity, - asyncApiFunc, - adminDelOpFunc, - topic_uri_fun, - enablePartionFunc, - partitionNumFunc); - } - - private LongFunction resolveMsgSend( - PulsarSpace clientSpace, - LongFunction topic_uri_func, - LongFunction async_api_func, - LongFunction useTransactionFunc, - LongFunction seqTrackingFunc - ) { - LongFunction> transactionSupplierFunc = - (l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle? - - LongFunction cycle_producer_name_func = lookupParameterFunc("producer_name"); - - LongFunction> producerFunc = - (l) -> clientSpace.getProducer(topic_uri_func.apply(l), cycle_producer_name_func.apply(l)); - - // check if we're going to simulate producer message out-of-sequence error - // - message ordering - // - message loss - Set seqErrSimuTypes = parseSimulatedErrorTypes(lookupStaticParameter("seqerr_simu")); - - // message key - LongFunction keyFunc = lookupParameterFunc("msg_key"); - - // message property - LongFunction propFunc = lookupParameterFunc("msg_property"); - - LongFunction valueFunc = lookupParameterFunc("msg_value", true); - - return new PulsarProducerMapper( - cmdTpl, - clientSpace, - pulsarActivity, - async_api_func, - useTransactionFunc, - seqTrackingFunc, - transactionSupplierFunc, - producerFunc, - seqErrSimuTypes, - keyFunc, - propFunc, - valueFunc); - } - - private Set parseSimulatedErrorTypes(String sequenceErrorSimulatedTypeString) { - if (StringUtils.isBlank(sequenceErrorSimulatedTypeString)) { - return Collections.emptySet(); - } - return Arrays.stream(StringUtils.split(sequenceErrorSimulatedTypeString, ',')) - .map(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE::parseSimuType) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } - - private LongFunction resolveMsgConsume( - PulsarSpace clientSpace, - LongFunction topic_uri_func, - LongFunction async_api_func, - LongFunction useTransactionFunc, - LongFunction seqTrackingFunc, - EndToEndStartingTimeSource endToEndStartingTimeSource, - LongFunction rttTrackingFieldFunc - ) { - LongFunction subscription_name_func = lookupParameterFunc("subscription_name"); - - LongFunction subscription_type_func = lookupParameterFunc("subscription_type"); - - LongFunction consumer_name_func = lookupParameterFunc("consumer_name"); - - LongFunction ranges_func = lookupParameterFunc("ranges", false, ""); - - LongFunction> transactionSupplierFunc = - (l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle? - - LongFunction> consumerFunc = (l) -> - clientSpace.getConsumer( - topic_uri_func.apply(l), - subscription_name_func.apply(l), - subscription_type_func.apply(l), - consumer_name_func.apply(l), - ranges_func.apply(l) - ); - - return new PulsarConsumerMapper( - cmdTpl, - clientSpace, - pulsarActivity, - async_api_func, - useTransactionFunc, - seqTrackingFunc, - transactionSupplierFunc, - consumerFunc, - endToEndStartingTimeSource, - rttTrackingFieldFunc); - } - - private LongFunction resolveMultiTopicMsgConsume( - PulsarSpace clientSpace, - LongFunction topic_uri_func, - LongFunction async_api_func, - LongFunction useTransactionFunc, - LongFunction seqTrackingFunc, - LongFunction payloadRttFieldFunc - ) { - // Topic list (multi-topic) - LongFunction topic_names_func = lookupParameterFunc("topic_names"); - - // Topic pattern (multi-topic) - LongFunction topics_pattern_func = lookupParameterFunc("topics_pattern"); - - LongFunction subscription_name_func = lookupParameterFunc("subscription_name"); - - LongFunction subscription_type_func = lookupParameterFunc("subscription_type"); - - LongFunction consumer_name_func = lookupParameterFunc("consumer_name"); - - LongFunction> transactionSupplierFunc = - (l) -> clientSpace.getTransactionSupplier(); //TODO make it dependant on current cycle? - - LongFunction> mtConsumerFunc = (l) -> - clientSpace.getMultiTopicConsumer( - topic_uri_func.apply(l), - topic_names_func.apply(l), - topics_pattern_func.apply(l), - subscription_name_func.apply(l), - subscription_type_func.apply(l), - consumer_name_func.apply(l) - ); - - return new PulsarConsumerMapper( - cmdTpl, - clientSpace, - pulsarActivity, - async_api_func, - useTransactionFunc, - seqTrackingFunc, - transactionSupplierFunc, - mtConsumerFunc, - parseEndToEndStartingTimeSourceParameter(EndToEndStartingTimeSource.NONE), - payloadRttFieldFunc); - } - - private LongFunction lookupParameterFunc(String parameterName) { - return lookupParameterFunc(parameterName, false, null); - } - - private LongFunction lookupParameterFunc(String parameterName, boolean required) { - return lookupParameterFunc(parameterName, required, null); - } - - private LongFunction lookupParameterFunc(String parameterName, boolean required, String defaultValue) { - if (cmdTpl.containsKey(parameterName)) { - LongFunction lookupFunc; - if (cmdTpl.isStatic(parameterName)) { - String staticValue = cmdTpl.getStatic(parameterName); - lookupFunc = (l) -> staticValue; - } else if (cmdTpl.isDynamic(parameterName)) { - lookupFunc = (l) -> cmdTpl.getDynamic(parameterName, l); - } else { - lookupFunc = (l) -> defaultValue; - } - return lookupFunc; - } else { - if (required) { - throw new PulsarDriverParamException("\"" + parameterName + "\" field must be specified!"); - } else { - return (l) -> defaultValue; - } - } - } - - private String lookupStaticParameter(String parameterName) { - return lookupStaticParameter(parameterName, false, null); - } - - private String lookupStaticParameter(String parameterName, boolean required, String defaultValue) { - if (cmdTpl.containsKey(parameterName)) { - if (cmdTpl.isStatic(parameterName)) { - return cmdTpl.getStatic(parameterName); - } else if (cmdTpl.isDynamic(parameterName)) { - throw new PulsarDriverParamException("\"" + parameterName + "\" parameter must be static"); - } else { - return defaultValue; - } - } else { - if (required) { - throw new PulsarDriverParamException("\"" + parameterName + "\" field must be specified!"); - } else { - return defaultValue; - } - } - } - - private LongFunction toBooleanFunc(LongFunction parameterFunc) { - return (l) -> BooleanUtils.toBoolean(parameterFunc.apply(l)); - } - - private LongFunction resolveMsgRead( - PulsarSpace clientSpace, - LongFunction topic_uri_func, - LongFunction async_api_func - ) { - LongFunction reader_name_func = lookupParameterFunc("reader_name"); - - LongFunction start_msg_pos_str_func = lookupParameterFunc("start_msg_position"); - - LongFunction> readerFunc = (l) -> - clientSpace.getReader( - topic_uri_func.apply(l), - reader_name_func.apply(l), - start_msg_pos_str_func.apply(l) - ); - - return new PulsarReaderMapper( - cmdTpl, - clientSpace, - pulsarActivity, - async_api_func, - readerFunc); - } - - private LongFunction resolveMsgBatchSendStart( - PulsarSpace clientSpace, - LongFunction topic_uri_func, - LongFunction asyncApiFunc) - { - LongFunction cycle_batch_producer_name_func = lookupParameterFunc("batch_producer_name"); - - LongFunction> batchProducerFunc = - (l) -> clientSpace.getProducer(topic_uri_func.apply(l), cycle_batch_producer_name_func.apply(l)); - - return new PulsarBatchProducerStartMapper( - cmdTpl, - clientSpace, - pulsarActivity, - asyncApiFunc, - batchProducerFunc); - } - - private LongFunction resolveMsgBatchSend(PulsarSpace clientSpace, - LongFunction asyncApiFunc) - { - LongFunction keyFunc = lookupParameterFunc("msg_key"); - - // message property - LongFunction propFunc = lookupParameterFunc("msg_property"); - - LongFunction valueFunc = lookupParameterFunc("msg_value", true); - - return new PulsarBatchProducerMapper( - cmdTpl, - clientSpace, - pulsarActivity, - asyncApiFunc, - keyFunc, - propFunc, - valueFunc); - } - - private LongFunction resolveMsgBatchSendEnd(PulsarSpace clientSpace, - LongFunction asyncApiFunc) - { - return new PulsarBatchProducerEndMapper( - cmdTpl, - clientSpace, - pulsarActivity, - asyncApiFunc); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTracker.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTracker.java deleted file mode 100644 index 693531334..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTracker.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import com.codahale.metrics.Counter; -import java.util.Iterator; -import java.util.SortedSet; -import java.util.TreeSet; - -/** - * Detects message loss, message duplication and out-of-order message delivery - * based on a monotonic sequence number that each received message contains. - *

- * Out-of-order messages are detected with a maximum look behind of 1000 sequence number entries. - * This is currently defined as a constant, {@link ReceivedMessageSequenceTracker#DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS}. - */ -class ReceivedMessageSequenceTracker implements AutoCloseable { - private static final int DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS = 1000; - private static final int DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS = 1000; - // message out-of-sequence error counter - private final Counter msgErrOutOfSeqCounter; - // duplicate message error counter - private final Counter msgErrDuplicateCounter; - // message loss error counter - private final Counter msgErrLossCounter; - private final SortedSet pendingOutOfSeqNumbers; - private final int maxTrackOutOfOrderSequenceNumbers; - private final SortedSet skippedSeqNumbers; - private final int maxTrackSkippedSequenceNumbers; - private long expectedNumber = -1; - - public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter) { - this(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter, - DEFAULT_MAX_TRACK_OUT_OF_ORDER_SEQUENCE_NUMBERS, DEFAULT_MAX_TRACK_SKIPPED_SEQUENCE_NUMBERS); - } - - public ReceivedMessageSequenceTracker(Counter msgErrOutOfSeqCounter, Counter msgErrDuplicateCounter, Counter msgErrLossCounter, - int maxTrackOutOfOrderSequenceNumbers, int maxTrackSkippedSequenceNumbers) { - this.msgErrOutOfSeqCounter = msgErrOutOfSeqCounter; - this.msgErrDuplicateCounter = msgErrDuplicateCounter; - this.msgErrLossCounter = msgErrLossCounter; - this.maxTrackOutOfOrderSequenceNumbers = maxTrackOutOfOrderSequenceNumbers; - this.maxTrackSkippedSequenceNumbers = maxTrackSkippedSequenceNumbers; - this.pendingOutOfSeqNumbers = new TreeSet<>(); - this.skippedSeqNumbers = new TreeSet<>(); - } - - /** - * Notifies the tracker about a received sequence number - * - * @param sequenceNumber the sequence number of the received message - */ - public void sequenceNumberReceived(long sequenceNumber) { - if (expectedNumber == -1) { - expectedNumber = sequenceNumber + 1; - return; - } - - if (sequenceNumber < expectedNumber) { - if (skippedSeqNumbers.remove(sequenceNumber)) { - // late out-of-order delivery was detected - // decrease the loss counter - msgErrLossCounter.dec(); - // increment the out-of-order counter - msgErrOutOfSeqCounter.inc(); - } else { - msgErrDuplicateCounter.inc(); - } - return; - } - - boolean messagesSkipped = false; - if (sequenceNumber > expectedNumber) { - if (pendingOutOfSeqNumbers.size() == maxTrackOutOfOrderSequenceNumbers) { - messagesSkipped = processLowestPendingOutOfSequenceNumber(); - } - if (!pendingOutOfSeqNumbers.add(sequenceNumber)) { - msgErrDuplicateCounter.inc(); - } - } else { - // sequenceNumber == expectedNumber - expectedNumber++; - } - processPendingOutOfSequenceNumbers(messagesSkipped); - cleanUpTooFarBehindOutOfSequenceNumbers(); - } - - private boolean processLowestPendingOutOfSequenceNumber() { - // remove the lowest pending out of sequence number - Long lowestOutOfSeqNumber = pendingOutOfSeqNumbers.first(); - pendingOutOfSeqNumbers.remove(lowestOutOfSeqNumber); - if (lowestOutOfSeqNumber > expectedNumber) { - // skip the expected number ahead to the number after the lowest sequence number - // increment the counter with the amount of sequence numbers that got skipped - // keep track of the skipped sequence numbers to detect late out-of-order message delivery - for (long l = expectedNumber; l < lowestOutOfSeqNumber; l++) { - msgErrLossCounter.inc(); - skippedSeqNumbers.add(l); - if (skippedSeqNumbers.size() > maxTrackSkippedSequenceNumbers) { - skippedSeqNumbers.remove(skippedSeqNumbers.first()); - } - } - expectedNumber = lowestOutOfSeqNumber + 1; - return true; - } else { - msgErrLossCounter.inc(); - } - return false; - } - - private void processPendingOutOfSequenceNumbers(boolean messagesSkipped) { - // check if there are previously received out-of-order sequence number that have been received - while (pendingOutOfSeqNumbers.remove(expectedNumber)) { - expectedNumber++; - if (!messagesSkipped) { - msgErrOutOfSeqCounter.inc(); - } - } - } - - private void cleanUpTooFarBehindOutOfSequenceNumbers() { - // remove sequence numbers that are too far behind - for (Iterator iterator = pendingOutOfSeqNumbers.iterator(); iterator.hasNext(); ) { - Long number = iterator.next(); - if (number < expectedNumber - maxTrackOutOfOrderSequenceNumbers) { - msgErrLossCounter.inc(); - iterator.remove(); - } else { - break; - } - } - } - - /** - * Handles the possible pending out of sequence numbers. Mainly needed in unit tests to assert the - * counter values. - */ - @Override - public void close() { - while (!pendingOutOfSeqNumbers.isEmpty()) { - processPendingOutOfSequenceNumbers(processLowestPendingOutOfSequenceNumber()); - } - } - - public int getMaxTrackOutOfOrderSequenceNumbers() { - return maxTrackOutOfOrderSequenceNumbers; - } - - public int getMaxTrackSkippedSequenceNumbers() { - return maxTrackSkippedSequenceNumbers; - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/SyncPulsarOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/SyncPulsarOp.java deleted file mode 100644 index 417a4a7b0..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/SyncPulsarOp.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -/** - * Base type of all Sync Pulsar Operations including Producers and Consumers. - */ -public abstract class SyncPulsarOp implements PulsarOp { - - public void run(Runnable timeTracker) { - try { - this.run(); - } finally { - timeTracker.run(); - } - } - - public abstract void run(); -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/AvroUtil.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/AvroUtil.java deleted file mode 100644 index eb101bf1d..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/AvroUtil.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.util; - -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.JsonDecoder; -import org.apache.avro.io.BinaryDecoder; -import org.apache.pulsar.client.api.schema.Field; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.GenericRecordBuilder; -import org.apache.pulsar.client.impl.schema.SchemaInfoImpl; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; - -public class AvroUtil { - //////////////////////// - // Get an OSS Apache Avro schema from a string definition - public static org.apache.avro.Schema GetSchema_ApacheAvro(String avroSchemDef) { - return new org.apache.avro.Schema.Parser().parse(avroSchemDef); - } - - // Get an OSS Apache Avro schema record from a JSON string that matches a specific OSS Apache Avro schema - public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, String jsonData) { - org.apache.avro.generic.GenericRecord record = null; - - try { - org.apache.avro.generic.GenericDatumReader reader; - reader = new org.apache.avro.generic.GenericDatumReader<>(schema); - - JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, jsonData); - - record = reader.read(null, decoder); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - - return record; - } - - // Get an OSS Apache Avro schema record from a byte array that matches a specific OSS Apache Avro schema - public static org.apache.avro.generic.GenericRecord GetGenericRecord_ApacheAvro(org.apache.avro.Schema schema, byte[] bytesData) { - org.apache.avro.generic.GenericRecord record = null; - - try { - org.apache.avro.generic.GenericDatumReader reader; - reader = new org.apache.avro.generic.GenericDatumReader<>(schema); - - BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytesData, null); - - record = reader.read(null, decoder); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - - return record; - } - - - //////////////////////// - // Get a Pulsar Avro schema from a string definition - public static GenericAvroSchema GetSchema_PulsarAvro(String schemaName, String avroSchemDef) { - SchemaInfo schemaInfo = SchemaInfoImpl.builder() - .schema(avroSchemDef.getBytes(StandardCharsets.UTF_8)) - .type(SchemaType.AVRO) - .properties(new HashMap<>()) - .name(schemaName) - .build(); - return new GenericAvroSchema(schemaInfo); - } - - // Get a Pulsar Avro record from an OSS Avro schema record, matching a specific Pulsar Avro schema - public static GenericRecord GetGenericRecord_PulsarAvro( - GenericAvroSchema pulsarGenericAvroSchema, - org.apache.avro.generic.GenericRecord apacheAvroGenericRecord) - { - GenericRecordBuilder recordBuilder = pulsarGenericAvroSchema.newRecordBuilder(); - - List fieldList = pulsarGenericAvroSchema.getFields(); - for (Field field : fieldList) { - String fieldName = field.getName(); - recordBuilder.set(fieldName, apacheAvroGenericRecord.get(fieldName)); - } - - return recordBuilder.build(); - } - - // Get a Pulsar Avro record (GenericRecord) from a JSON string that matches a specific Pulsar Avro schema - public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, String avroSchemDefStr, String jsonData) { - org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr); - return GetGenericRecord_PulsarAvro(genericAvroSchema, avroSchema, jsonData); - } - - public static GenericRecord GetGenericRecord_PulsarAvro(GenericAvroSchema genericAvroSchema, org.apache.avro.Schema avroSchema, String jsonData) { - org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData); - return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord); - } - public static GenericRecord GetGenericRecord_PulsarAvro(String schemaName, String avroSchemDefStr, String jsonData) { - GenericAvroSchema genericAvroSchema = GetSchema_PulsarAvro(schemaName, avroSchemDefStr); - org.apache.avro.Schema avroSchema = GetSchema_ApacheAvro(avroSchemDefStr); - org.apache.avro.generic.GenericRecord apacheAvroRecord = GetGenericRecord_ApacheAvro(avroSchema, jsonData); - - return GetGenericRecord_PulsarAvro(genericAvroSchema, apacheAvroRecord); - } -} diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarActivityUtil.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarActivityUtil.java deleted file mode 100644 index 6adb24f40..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarActivityUtil.java +++ /dev/null @@ -1,535 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.util; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pulsar.client.api.Schema; - -import java.io.IOException; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class PulsarActivityUtil { - - private final static Logger logger = LogManager.getLogger(PulsarActivityUtil.class); - - // Supported message operation types - // TODO: websocket-producer and managed-ledger - public enum OP_TYPES { - ADMIN_TENANT("admin-tenant"), - ADMIN_NAMESPACE("admin-namespace"), - ADMIN_TOPIC("admin-topic"), - E2E_MSG_PROC_SEND("e22-msg-proc-send"), - E2E_MSG_PROC_CONSUME("e22-msg-proc-consume"), -// BATCH_MSG_SEND_START("batch-msg-send-start"), -// BATCH_MSG_SEND("batch-msg-send"), -// BATCH_MSG_SEND_END("batch-msg-send-end"), - MSG_SEND("msg-send"), - MSG_CONSUME("msg-consume"), - MSG_READ("msg-read"), - MSG_MULTI_CONSUME("msg-mt-consume"); - - public final String label; - - OP_TYPES(String label) { - this.label = label; - } - } - - - public static boolean isValidClientType(String type) { - return Arrays.stream(OP_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - - public static final String MSG_SEQUENCE_NUMBER = "sequence_number"; - - /////// - // Valid document level parameters for Pulsar NB yaml file - public enum DOC_LEVEL_PARAMS { - TOPIC_URI("topic_uri"), - ASYNC_API("async_api"), - USE_TRANSACTION("use_transaction"), - ADMIN_DELOP("admin_delop"), - SEQ_TRACKING("seq_tracking"), - MSG_DEDUP_BROKER("msg_dedup_broker"), - E2E_STARTING_TIME_SOURCE("e2e_starting_time_source"); - - public final String label; - - DOC_LEVEL_PARAMS(String label) { - this.label = label; - } - } - public static boolean isValidDocLevelParam(String param) { - return Arrays.stream(DOC_LEVEL_PARAMS.values()).anyMatch(t -> t.label.equals(param)); - } - - /////// - // Valid Pulsar API type - public enum PULSAR_API_TYPE { - PRODUCER("producer"), - CONSUMER("consumer"), - READER("reader"); - - public final String label; - - PULSAR_API_TYPE(String label) { - this.label = label; - } - } - public static boolean isValidPulsarApiType(String param) { - return Arrays.stream(PULSAR_API_TYPE.values()).anyMatch(t -> t.label.equals(param)); - } - public static String getValidPulsarApiTypeList() { - return Arrays.stream(PULSAR_API_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); - } - - /////// - // Valid persistence type - public enum PERSISTENT_TYPES { - PERSISTENT("persistent"), - NON_PERSISTENT("non-persistent") - ; - - public final String label; - PERSISTENT_TYPES(String label) { - this.label = label; - } - } - public static boolean isValidPersistenceType(String type) { - return Arrays.stream(PERSISTENT_TYPES.values()).anyMatch(t -> t.label.equals(type)); - } - - /////// - // Valid Pulsar client configuration (activity-level settings) - // - https://pulsar.apache.org/docs/en/client-libraries-java/#client - public enum CLNT_CONF_KEY { - serviceUrl("serviceUrl"), - authPulginClassName("authPluginClassName"), - authParams("authParams"), - pperationTimeoutMs("operationTimeoutMs"), - statsIntervalSeconds("statsIntervalSeconds"), - numIoThreads("numIoThreads"), - numListenerThreads("numListenerThreads"), - useTcpNoDelay("useTcpNoDelay"), - useTls("useTls"), - tlsTrustCertsFilePath("tlsTrustCertsFilePath"), - tlsAllowInsecureConnection("tlsAllowInsecureConnection"), - tlsHostnameVerificationEnable("tlsHostnameVerificationEnable"), - concurrentLookupRequest("concurrentLookupRequest"), - maxLookupRequest("maxLookupRequest"), - maxNumberOfRejectedRequestPerConnection("maxNumberOfRejectedRequestPerConnection"), - keepAliveIntervalSeconds("keepAliveIntervalSeconds"), - connectionTimeoutMs("connectionTimeoutMs"), - requestTimeoutMs("requestTimeoutMs"), - defaultBackoffIntervalNanos("defaultBackoffIntervalNanos"), - maxBackoffIntervalNanos("maxBackoffIntervalNanos") - ; - - public final String label; - CLNT_CONF_KEY(String label) { - this.label = label; - } - } - public static boolean isValidClientConfItem(String item) { - return Arrays.stream(CLNT_CONF_KEY.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Standard producer configuration (activity-level settings) - // - https://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer - public enum PRODUCER_CONF_STD_KEY { - topicName("topicName"), - producerName("producerName"), - sendTimeoutMs("sendTimeoutMs"), - blockIfQueueFull("blockIfQueueFull"), - maxPendingMessages("maxPendingMessages"), - maxPendingMessagesAcrossPartitions("maxPendingMessagesAcrossPartitions"), - messageRoutingMode("messageRoutingMode"), - hashingScheme("hashingScheme"), - cryptoFailureAction("cryptoFailureAction"), - batchingMaxPublishDelayMicros("batchingMaxPublishDelayMicros"), - batchingMaxMessages("batchingMaxMessages"), - batchingEnabled("batchingEnabled"), - compressionType("compressionType"); - - public final String label; - - PRODUCER_CONF_STD_KEY(String label) { - this.label = label; - } - } - public static boolean isStandardProducerConfItem(String item) { - return Arrays.stream(PRODUCER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Standard consumer configuration (activity-level settings) - // - https://pulsar.apache.org/docs/en/client-libraries-java/#consumer - public enum CONSUMER_CONF_STD_KEY { - topicNames("topicNames"), - topicsPattern("topicsPattern"), - subscriptionName("subscriptionName"), - subscriptionType("subscriptionType"), - receiverQueueSize("receiverQueueSize"), - acknowledgementsGroupTimeMicros("acknowledgementsGroupTimeMicros"), - negativeAckRedeliveryDelayMicros("negativeAckRedeliveryDelayMicros"), - maxTotalReceiverQueueSizeAcrossPartitions("maxTotalReceiverQueueSizeAcrossPartitions"), - consumerName("consumerName"), - ackTimeoutMillis("ackTimeoutMillis"), - tickDurationMillis("tickDurationMillis"), - priorityLevel("priorityLevel"), - cryptoFailureAction("cryptoFailureAction"), - properties("properties"), - readCompacted("readCompacted"), - subscriptionInitialPosition("subscriptionInitialPosition"), - patternAutoDiscoveryPeriod("patternAutoDiscoveryPeriod"), - regexSubscriptionMode("regexSubscriptionMode"), - deadLetterPolicy("deadLetterPolicy"), - autoUpdatePartitions("autoUpdatePartitions"), - replicateSubscriptionState("replicateSubscriptionState"); - - public final String label; - - CONSUMER_CONF_STD_KEY(String label) { - this.label = label; - } - } - public static boolean isStandardConsumerConfItem(String item) { - return Arrays.stream(CONSUMER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Custom consumer configuration (activity-level settings) - // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#consumer - // - NB Pulsar driver consumer operation specific - public enum CONSUMER_CONF_CUSTOM_KEY { - timeout("timeout"); - - public final String label; - - CONSUMER_CONF_CUSTOM_KEY(String label) { - this.label = label; - } - } - public static boolean isCustomConsumerConfItem(String item) { - return Arrays.stream(CONSUMER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Pulsar subscription type - public enum SUBSCRIPTION_TYPE { - Exclusive("Exclusive"), - Failover("Failover"), - Shared("Shared"), - Key_Shared("Key_Shared"); - - public final String label; - - SUBSCRIPTION_TYPE(String label) { - this.label = label; - } - } - public static boolean isValidSubscriptionType(String item) { - return Arrays.stream(SUBSCRIPTION_TYPE.values()).anyMatch(t -> t.label.equals(item)); - } - public static String getValidSubscriptionTypeList() { - return Arrays.stream(SUBSCRIPTION_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); - } - - /////// - // Standard reader configuration (activity-level settings) - // - https://pulsar.apache.org/docs/en/client-libraries-java/#reader - public enum READER_CONF_STD_KEY { - topicName("topicName"), - receiverQueueSize("receiverQueueSize"), - readerListener("readerListener"), - readerName("readerName"), - subscriptionRolePrefix("subscriptionRolePrefix"), - cryptoKeyReader("cryptoKeyReader"), - cryptoFailureAction("cryptoFailureAction"), - readCompacted("readCompacted"), - resetIncludeHead("resetIncludeHead"); - - public final String label; - - READER_CONF_STD_KEY(String label) { - this.label = label; - } - } - public static boolean isStandardReaderConfItem(String item) { - return Arrays.stream(READER_CONF_STD_KEY.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Custom reader configuration (activity-level settings) - // - NOT part of https://pulsar.apache.org/docs/en/client-libraries-java/#reader - // - NB Pulsar driver reader operation specific - public enum READER_CONF_CUSTOM_KEY { - startMessagePos("startMessagePos"); - - public final String label; - - READER_CONF_CUSTOM_KEY(String label) { - this.label = label; - } - } - public static boolean isCustomReaderConfItem(String item) { - return Arrays.stream(READER_CONF_CUSTOM_KEY.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Valid read positions for a Pulsar reader - public enum READER_MSG_POSITION_TYPE { - earliest("earliest"), - latest("latest"), - custom("custom"); - - public final String label; - - READER_MSG_POSITION_TYPE(String label) { - this.label = label; - } - } - public static boolean isValideReaderStartPosition(String item) { - return Arrays.stream(READER_MSG_POSITION_TYPE.values()).anyMatch(t -> t.label.equals(item)); - } - - /////// - // Pulsar subscription type - public enum SEQ_ERROR_SIMU_TYPE { - OutOfOrder("out_of_order"), - MsgLoss("msg_loss"), - MsgDup("msg_dup"); - - public final String label; - - SEQ_ERROR_SIMU_TYPE(String label) { - this.label = label; - } - - private static final Map MAPPING = new HashMap<>(); - - static { - for (SEQ_ERROR_SIMU_TYPE simuType : values()) { - MAPPING.put(simuType.label, simuType); - MAPPING.put(simuType.label.toLowerCase(), simuType); - MAPPING.put(simuType.label.toUpperCase(), simuType); - MAPPING.put(simuType.name(), simuType); - MAPPING.put(simuType.name().toLowerCase(), simuType); - MAPPING.put(simuType.name().toUpperCase(), simuType); - } - } - - public static Optional parseSimuType(String simuTypeString) { - return Optional.ofNullable(MAPPING.get(simuTypeString.trim())); - } - } - public static boolean isValidSeqErrSimuType(String item) { - return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).anyMatch(t -> t.label.equals(item)); - } - public static String getValidSeqErrSimuTypeList() { - return Arrays.stream(SEQ_ERROR_SIMU_TYPE.values()).map(t -> t.label).collect(Collectors.joining(", ")); - } - - /////// - // Valid websocket-producer configuration (activity-level settings) - // TODO: to be added - public enum WEBSKT_PRODUCER_CONF_KEY { - ; - - public final String label; - - WEBSKT_PRODUCER_CONF_KEY(String label) { - this.label = label; - } - } - - /////// - // Valid managed-ledger configuration (activity-level settings) - // TODO: to be added - public enum MANAGED_LEDGER_CONF_KEY { - ; - - public final String label; - MANAGED_LEDGER_CONF_KEY(String label) { - this.label = label; - } - } - - /////// - // Primitive Schema type - public static boolean isPrimitiveSchemaTypeStr(String typeStr) { - boolean isPrimitive = false; - - // Use "BYTES" as the default type if the type string is not explicitly specified - if (StringUtils.isBlank(typeStr)) { - typeStr = "BYTES"; - } - - if (typeStr.equalsIgnoreCase("BOOLEAN") || typeStr.equalsIgnoreCase("INT8") || - typeStr.equalsIgnoreCase("INT16") || typeStr.equalsIgnoreCase("INT32") || - typeStr.equalsIgnoreCase("INT64") || typeStr.equalsIgnoreCase("FLOAT") || - typeStr.equalsIgnoreCase("DOUBLE") || typeStr.equalsIgnoreCase("BYTES") || - typeStr.equalsIgnoreCase("DATE") || typeStr.equalsIgnoreCase("TIME") || - typeStr.equalsIgnoreCase("TIMESTAMP") || typeStr.equalsIgnoreCase("INSTANT") || - typeStr.equalsIgnoreCase("LOCAL_DATE") || typeStr.equalsIgnoreCase("LOCAL_TIME") || - typeStr.equalsIgnoreCase("LOCAL_DATE_TIME")) { - isPrimitive = true; - } - - return isPrimitive; - } - public static Schema getPrimitiveTypeSchema(String typeStr) { - Schema schema; - - switch (typeStr.toUpperCase()) { - case "BOOLEAN": - schema = Schema.BOOL; - break; - case "INT8": - schema = Schema.INT8; - break; - case "INT16": - schema = Schema.INT16; - break; - case "INT32": - schema = Schema.INT32; - break; - case "INT64": - schema = Schema.INT64; - break; - case "FLOAT": - schema = Schema.FLOAT; - break; - case "DOUBLE": - schema = Schema.DOUBLE; - break; - case "DATE": - schema = Schema.DATE; - break; - case "TIME": - schema = Schema.TIME; - break; - case "TIMESTAMP": - schema = Schema.TIMESTAMP; - break; - case "INSTANT": - schema = Schema.INSTANT; - break; - case "LOCAL_DATE": - schema = Schema.LOCAL_DATE; - break; - case "LOCAL_TIME": - schema = Schema.LOCAL_TIME; - break; - case "LOCAL_DATE_TIME": - schema = Schema.LOCAL_DATE_TIME; - break; - // Use BYTES as the default schema type if the type string is not specified - case "": - case "BYTES": - schema = Schema.BYTES; - break; - // Report an error if non-valid, non-empty schema type string is provided - default: - throw new RuntimeException("Invalid Pulsar primitive schema type string : " + typeStr); - } - - return schema; - } - - /////// - // Complex strut type: Avro or Json - public static boolean isAvroSchemaTypeStr(String typeStr) { - return typeStr.equalsIgnoreCase("AVRO"); - } - public static boolean isKeyValueTypeStr(String typeStr) { - return typeStr.equalsIgnoreCase("KEY_VALUE"); - } - - // automatic decode the type from the Registry - public static boolean isAutoConsumeSchemaTypeStr(String typeStr) { - return typeStr.equalsIgnoreCase("AUTO_CONSUME"); - } - public static Schema getAvroSchema(String typeStr, String definitionStr) { - String schemaDefinitionStr = definitionStr; - String filePrefix = "file://"; - Schema schema; - - // Check if payloadStr points to a file (e.g. "file:///path/to/a/file") - if (isAvroSchemaTypeStr(typeStr)) { - if (StringUtils.isBlank(schemaDefinitionStr)) { - throw new RuntimeException("Schema definition must be provided for \"Avro\" schema type!"); - } else if (schemaDefinitionStr.startsWith(filePrefix)) { - try { - Path filePath = Paths.get(URI.create(schemaDefinitionStr)); - schemaDefinitionStr = Files.readString(filePath, StandardCharsets.US_ASCII); - } catch (IOException ioe) { - throw new RuntimeException("Error reading the specified \"Avro\" schema definition file: " + definitionStr + ": " + ioe.getMessage()); - } - } - - schema = AvroUtil.GetSchema_PulsarAvro("NBAvro", schemaDefinitionStr); - } else { - throw new RuntimeException("Trying to create a \"Avro\" schema for a non-Avro schema type string: " + typeStr); - } - - return schema; - } - - /////// - // Generate effective key string - public static String buildCacheKey(String... keyParts) { - // Ignore blank keyPart - String joinedKeyStr = - Stream.of(keyParts) - .filter(s -> !StringUtils.isBlank(s)) - .collect(Collectors.joining(",")); - - return Base64.getEncoder().encodeToString(joinedKeyStr.getBytes()); - } - - /////// - // Convert JSON string to a key/value map - public static Map convertJsonToMap(String jsonStr) throws Exception { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(jsonStr, Map.class); - } - - /////// - // Get full namespace name (/) from a Pulsar topic URI - public static String getFullNamespaceName(String topicUri) { - // Get tenant/namespace string - // - topicUri : persistent://// - // - tmpStr : // - // - fullNsName : / - - String tmpStr = StringUtils.substringAfter(topicUri,"://"); - return StringUtils.substringBeforeLast(tmpStr, "/"); - } -} - diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java deleted file mode 100644 index eef8251c1..000000000 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.util; - -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.FileBasedConfiguration; -import org.apache.commons.configuration2.PropertiesConfiguration; -import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder; -import org.apache.commons.configuration2.builder.fluent.Parameters; -import org.apache.commons.configuration2.ex.ConfigurationException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class PulsarNBClientConf { - - private final static Logger logger = LogManager.getLogger(PulsarNBClientConf.class); - - private String canonicalFilePath = ""; - - public static final String SCHEMA_CONF_PREFIX = "schema"; - public static final String CLIENT_CONF_PREFIX = "client"; - public static final String PRODUCER_CONF_PREFIX = "producer"; - public static final String CONSUMER_CONF_PREFIX = "consumer"; - public static final String READER_CONF_PREFIX = "reader"; - private final HashMap schemaConfMap = new HashMap<>(); - private final HashMap clientConfMap = new HashMap<>(); - private final HashMap producerConfMap = new HashMap<>(); - private final HashMap consumerConfMap = new HashMap<>(); - private final HashMap readerConfMap = new HashMap<>(); - // TODO: add support for other operation types: websocket-producer, managed-ledger - - public PulsarNBClientConf(String fileName) { - File file = new File(fileName); - - try { - canonicalFilePath = file.getCanonicalPath(); - - Parameters params = new Parameters(); - - FileBasedConfigurationBuilder builder = - new FileBasedConfigurationBuilder(PropertiesConfiguration.class) - .configure(params.properties() - .setFileName(fileName)); - - Configuration config = builder.getConfiguration(); - - // Get schema specific configuration settings - for (Iterator it = config.getKeys(SCHEMA_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - schemaConfMap.put(confKey.substring(SCHEMA_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get client connection specific configuration settings - for (Iterator it = config.getKeys(CLIENT_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - clientConfMap.put(confKey.substring(CLIENT_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get producer specific configuration settings - for (Iterator it = config.getKeys(PRODUCER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - producerConfMap.put(confKey.substring(PRODUCER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get consumer specific configuration settings - for (Iterator it = config.getKeys(CONSUMER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - consumerConfMap.put(confKey.substring(CONSUMER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - - // Get reader specific configuration settings - for (Iterator it = config.getKeys(READER_CONF_PREFIX); it.hasNext(); ) { - String confKey = it.next(); - String confVal = config.getProperty(confKey).toString(); - if (!StringUtils.isBlank(confVal)) - readerConfMap.put(confKey.substring(READER_CONF_PREFIX.length() + 1), config.getProperty(confKey)); - } - } catch (IOException ioe) { - logger.error("Can't read the specified config properties file!"); - ioe.printStackTrace(); - } catch (ConfigurationException cex) { - logger.error("Error loading configuration items from the specified config properties file: " + canonicalFilePath); - cex.printStackTrace(); - } - } - - - ////////////////// - // Get Schema related config - public Map getSchemaConfMap() { - return this.schemaConfMap; - } - public boolean hasSchemaConfKey(String key) { - if (key.contains(SCHEMA_CONF_PREFIX)) - return schemaConfMap.containsKey(key.substring(SCHEMA_CONF_PREFIX.length() + 1)); - else - return schemaConfMap.containsKey(key); - } - public Object getSchemaConfValue(String key) { - if (key.contains(SCHEMA_CONF_PREFIX)) - return schemaConfMap.get(key.substring(SCHEMA_CONF_PREFIX.length()+1)); - else - return schemaConfMap.get(key); - } - public void setSchemaConfValue(String key, Object value) { - if (key.contains(SCHEMA_CONF_PREFIX)) - schemaConfMap.put(key.substring(SCHEMA_CONF_PREFIX.length() + 1), value); - else - schemaConfMap.put(key, value); - } - - - ////////////////// - // Get Pulsar client related config - public Map getClientConfMap() { - return this.clientConfMap; - } - public boolean hasClientConfKey(String key) { - if (key.contains(CLIENT_CONF_PREFIX)) - return clientConfMap.containsKey(key.substring(CLIENT_CONF_PREFIX.length() + 1)); - else - return clientConfMap.containsKey(key); - } - public Object getClientConfValue(String key) { - if (key.contains(CLIENT_CONF_PREFIX)) - return clientConfMap.get(key.substring(CLIENT_CONF_PREFIX.length()+1)); - else - return clientConfMap.get(key); - } - public void setClientConfValue(String key, Object value) { - if (key.contains(CLIENT_CONF_PREFIX)) - clientConfMap.put(key.substring(CLIENT_CONF_PREFIX.length() + 1), value); - else - clientConfMap.put(key, value); - } - - - ////////////////// - // Get Pulsar producer related config - public Map getProducerConfMap() { - return this.producerConfMap; - } - public boolean hasProducerConfKey(String key) { - if (key.contains(PRODUCER_CONF_PREFIX)) - return producerConfMap.containsKey(key.substring(PRODUCER_CONF_PREFIX.length() + 1)); - else - return producerConfMap.containsKey(key); - } - public Object getProducerConfValue(String key) { - if (key.contains(PRODUCER_CONF_PREFIX)) - return producerConfMap.get(key.substring(PRODUCER_CONF_PREFIX.length()+1)); - else - return producerConfMap.get(key); - } - public void setProducerConfValue(String key, Object value) { - if (key.contains(PRODUCER_CONF_PREFIX)) - producerConfMap.put(key.substring(PRODUCER_CONF_PREFIX.length()+1), value); - else - producerConfMap.put(key, value); - } - // other producer helper functions ... - public String getProducerName() { - Object confValue = getProducerConfValue( - "producer." + PulsarActivityUtil.PRODUCER_CONF_STD_KEY.producerName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getProducerTopicName() { - Object confValue = getProducerConfValue( - "producer." + PulsarActivityUtil.PRODUCER_CONF_STD_KEY.topicName); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - - - ////////////////// - // Get Pulsar consumer related config - public Map getConsumerConfMap() { - return this.consumerConfMap; - } - public boolean hasConsumerConfKey(String key) { - if (key.contains(CONSUMER_CONF_PREFIX)) - return consumerConfMap.containsKey(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); - else - return consumerConfMap.containsKey(key); - } - public Object getConsumerConfValue(String key) { - if (key.contains(CONSUMER_CONF_PREFIX)) - return consumerConfMap.get(key.substring(CONSUMER_CONF_PREFIX.length() + 1)); - else - return consumerConfMap.get(key); - } - public void setConsumerConfValue(String key, Object value) { - if (key.contains(CONSUMER_CONF_PREFIX)) - consumerConfMap.put(key.substring(CONSUMER_CONF_PREFIX.length() + 1), value); - else - consumerConfMap.put(key, value); - } - // Other consumer helper functions ... - public String getConsumerTopicNames() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicNames.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerTopicPattern() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.topicsPattern.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerSubscriptionName() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerSubscriptionType() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.subscriptionType.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getConsumerName() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarActivityUtil.CONSUMER_CONF_STD_KEY.consumerName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - // NOTE: Below are not a standard Pulsar consumer configuration parameter as - // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer" - // They're custom-made configuration properties for NB pulsar driver consumer. - public int getConsumerTimeoutSeconds() { - Object confValue = getConsumerConfValue( - "consumer." + PulsarActivityUtil.CONSUMER_CONF_CUSTOM_KEY.timeout.label); - if (confValue == null) - return -1; // infinite - else - return Integer.parseInt(confValue.toString()); - } - - ////////////////// - // Get Pulsar reader related config - public Map getReaderConfMap() { - return this.readerConfMap; - } - public boolean hasReaderConfKey(String key) { - if (key.contains(READER_CONF_PREFIX)) - return readerConfMap.containsKey(key.substring(READER_CONF_PREFIX.length() + 1)); - else - return readerConfMap.containsKey(key); - } - public Object getReaderConfValue(String key) { - if (key.contains(READER_CONF_PREFIX)) - return readerConfMap.get(key.substring(READER_CONF_PREFIX.length() + 1)); - else - return readerConfMap.get(key); - } - public void setReaderConfValue(String key, Object value) { - if (key.contains(READER_CONF_PREFIX)) - readerConfMap.put(key.substring(READER_CONF_PREFIX.length() + 1), value); - else - readerConfMap.put(key, value); - } - // Other reader helper functions ... - public String getReaderTopicName() { - Object confValue = getReaderConfValue( - "reader." + PulsarActivityUtil.READER_CONF_STD_KEY.topicName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - public String getReaderName() { - Object confValue = getReaderConfValue( - "reader." + PulsarActivityUtil.READER_CONF_STD_KEY.readerName.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } - // NOTE: Below are not a standard Pulsar reader configuration parameter as - // listed in "https://pulsar.apache.org/docs/en/client-libraries-java/#reader" - // They're custom-made configuration properties for NB pulsar driver reader. - public String getStartMsgPosStr() { - Object confValue = getReaderConfValue( - "reader." + PulsarActivityUtil.READER_CONF_CUSTOM_KEY.startMessagePos.label); - if (confValue == null) - return ""; - else - return confValue.toString(); - } -} diff --git a/driver-pulsar/src/main/resources/activities/bindingtest.yaml b/driver-pulsar/src/main/resources/activities/bindingtest.yaml deleted file mode 100644 index ddf25bf04..000000000 --- a/driver-pulsar/src/main/resources/activities/bindingtest.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Results: -# - 10 tenants -# - 2 namespaces per tanant -# - 5 topics per namespace -#------------------------------------------------------ -#tenant=tenant_0 namespace=default_0 core_topic_name=t0 -#tenant=tenant_0 namespace=default_0 core_topic_name=t1 -#tenant=tenant_0 namespace=default_0 core_topic_name=t2 -#tenant=tenant_0 namespace=default_0 core_topic_name=t3 -#tenant=tenant_0 namespace=default_0 core_topic_name=t4 -#tenant=tenant_0 namespace=default_1 core_topic_name=t0 -#tenant=tenant_0 namespace=default_1 core_topic_name=t1 -#tenant=tenant_0 namespace=default_1 core_topic_name=t2 -#tenant=tenant_0 namespace=default_1 core_topic_name=t3 -#tenant=tenant_0 namespace=default_1 core_topic_name=t4 -#tenant=tenant_1 namespace=default_0 core_topic_name=t0 -#tenant=tenant_1 namespace=default_0 core_topic_name=t1 -#tenant=tenant_1 namespace=default_0 core_topic_name=t2 -#tenant=tenant_1 namespace=default_0 core_topic_name=t3 -#tenant=tenant_1 namespace=default_0 core_topic_name=t4 -#tenant=tenant_1 namespace=default_1 core_topic_name=t0 -#tenant=tenant_1 namespace=default_1 core_topic_name=t1 -#tenant=tenant_1 namespace=default_1 core_topic_name=t2 -#tenant=tenant_1 namespace=default_1 core_topic_name=t3 -#tenant=tenant_1 namespace=default_1 core_topic_name=t4 -# ... ... - -bindings: - # message key and value - #mykey: NumberNameToString() - #myvalue: AlphaNumericString(20) - # Admin API - create tenant, namespace, and topic - tenant: Mod(100); Div(10L); ToString(); Prefix("tenant_") - namespace: Mod(10); Div(5L); ToString(); Prefix("default_") - core_topic_name: Mod(5); ToString(); Prefix("t") diff --git a/driver-pulsar/src/main/resources/activities/config.properties b/driver-pulsar/src/main/resources/activities/config.properties deleted file mode 100644 index 37be64384..000000000 --- a/driver-pulsar/src/main/resources/activities/config.properties +++ /dev/null @@ -1,52 +0,0 @@ -### Schema related configurations - schema.xxx -# valid types: -# - primitive type (https://pulsar.apache.org/docs/en/schema-understand/#primitive-type) -# - keyvalue (https://pulsar.apache.org/docs/en/schema-understand/#keyvalue) -# - strut (complex type) (https://pulsar.apache.org/docs/en/schema-understand/#struct) -# avro, json, protobuf -# -# TODO: as a starting point, only supports the following types -# 1) primitive types, including bytearray (byte[]) which is default, for messages without schema -# 2) Avro for messages with schema -#schema.type=avro -#schema.definition=file:///Users/yabinmeng/DataStax/MyNoSQLBench/nosqlbench/driver-pulsar/src/main/resources/activities/iot-example.avsc -schema.type= -schema.definition= - - -### Pulsar client related configurations - client.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#client -client.connectionTimeoutMs=5000 -client.authPluginClassName=org.apache.pulsar.client.impl.auth.AuthenticationToken -# Cluster admin -client.authParams= -client.tlsAllowInsecureConnection=true - - -### Producer related configurations (global) - producer.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-producer -producer.producerName= -producer.topicName= -producer.sendTimeoutMs= -producer.blockIfQueueFull=true -producer.maxPendingMessages=5000 -producer.batchingMaxMessages=5000 - - -### Consumer related configurations (global) - consumer.xxx -# http://pulsar.apache.org/docs/en/client-libraries-java/#configure-consumer -consumer.topicNames= -consumer.topicsPattern= -consumer.subscriptionName= -consumer.subscriptionType= -consumer.consumerName= -consumer.receiverQueueSize= - - -### Reader related configurations (global) - reader.xxx -# https://pulsar.apache.org/docs/en/client-libraries-java/#reader -# - valid Pos: earliest, latest, custom::file://// -reader.topicName= -reader.receiverQueueSize= -reader.readerName= -reader.startMessagePos=earliest diff --git a/driver-pulsar/src/main/resources/activities/iot-example.avsc b/driver-pulsar/src/main/resources/activities/iot-example.avsc deleted file mode 100644 index 20bb894fd..000000000 --- a/driver-pulsar/src/main/resources/activities/iot-example.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "record", - "name": "IotSensor", - "namespace": "TestNS", - "fields" : [ - {"name": "SensorID", "type": "string"}, - {"name": "SensorType", "type": "string"}, - {"name": "ReadingTime", "type": "string"}, - {"name": "ReadingValue", "type": "float"} - ] -} diff --git a/driver-pulsar/src/main/resources/activities/pulsar_admin_namespace.yaml b/driver-pulsar/src/main/resources/activities/pulsar_admin_namespace.yaml deleted file mode 100644 index 0a75413d3..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_admin_namespace.yaml +++ /dev/null @@ -1,22 +0,0 @@ -bindings: - # 20 namespaces: 10 tenants, 2 namespaces/tenant - tenant: Mod(20); Div(2L); ToString(); Prefix("tnt") - namespace: Mod(2); ToString(); Prefix("ns") - -params: - # "true" - asynchronous Pulsar Admin API - # "false" - synchronous Pulsar Admin API - async_api: "false" - # "true" - delete namespace - # "false" - create namespace - admin_delop: "false" - -blocks: - - name: create-namespace-block - tags: - phase: admin-namespace - admin_task: true - statements: - - name: s1 - optype: admin-namespace - namespace: "{tenant}/{namespace}" diff --git a/driver-pulsar/src/main/resources/activities/pulsar_admin_tenant.yaml b/driver-pulsar/src/main/resources/activities/pulsar_admin_tenant.yaml deleted file mode 100644 index 64c7bcb54..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_admin_tenant.yaml +++ /dev/null @@ -1,23 +0,0 @@ -bindings: - # 10 tenants - tenant: Mod(10); ToString(); Prefix("tnt") - -params: - # "true" - asynchronous Pulsar Admin API - # "false" - synchronous Pulsar Admin API - async_api: "false" - # "true" - delete tenant - # "false" - create tenant - admin_delop: "true" - -blocks: - - name: create-tenant-block - tags: - phase: admin-tenant - admin_task: true - statements: - - name: s1 - optype: admin-tenant -# admin_roles: -# allowed_clusters: - tenant: "{tenant}" diff --git a/driver-pulsar/src/main/resources/activities/pulsar_admin_topic.yaml b/driver-pulsar/src/main/resources/activities/pulsar_admin_topic.yaml deleted file mode 100644 index 5d2dd567d..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_admin_topic.yaml +++ /dev/null @@ -1,25 +0,0 @@ -bindings: - # 100 topics: 10 tenants, 2 namespaces/tenant, 5 topics/namespace - tenant: Mod(100); Div(10L); ToString(); Prefix("tnt") - namespace: Mod(10); Div(5L); ToString(); Prefix("ns") - core_topic_name: Mod(5); ToString(); Prefix("t") - -params: - topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}" - # "true" - asynchronous Pulsar Admin API - # "false" - synchronous Pulsar Admin API - async_api: "false" - # "true" - delete topic - # "false" - create topic - admin_delop: "false" - -blocks: - - name: create-topic-block - tags: - phase: admin-topic - admin_task: true - statements: - - name: s1 - optype: admin-topic - enable_partition: "false" - partition_num: "5" diff --git a/driver-pulsar/src/main/resources/activities/pulsar_client_avro.yaml b/driver-pulsar/src/main/resources/activities/pulsar_client_avro.yaml deleted file mode 100644 index 21d07b25e..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_client_avro.yaml +++ /dev/null @@ -1,93 +0,0 @@ -bindings: - # message key and value - mykey: NumberNameToString() - sensor_id: ToUUID();ToString(); - reading_time: ToDateTime(); - reading_value: ToFloat(100); - tenant: Mod(100); Div(10L); ToString(); Prefix("tnt") - namespace: Mod(10); Div(5L); ToString(); Prefix("ns") - core_topic_name: Mod(5); ToString(); Prefix("t") - -# document level parameters that apply to all Pulsar client types: -params: - topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}" -# topic_uri: "persistent://public/default/mytopic" - async_api: "true" - -blocks: - - name: batch-producer-block - tags: - phase: batch-producer - admin_task: false - statements: - - name: s1 - optype: batch-msg-send-start - # For batch producer, "producer_name" should be associated with batch start - # batch_producer_name: {batch_producer_name} - ratio: 1 - - name: s2 - optype: batch-msg-send - msg_key: "{mykey}" - msg_value: | - { - "SensorID": "{sensor_id}", - "SensorType": "Temperature", - "ReadingTime": "{reading_time}", - "ReadingValue": {reading_value} - } - ratio: 100 - - name: s3 - optype: batch-msg-send-end - ratio: 1 - - - name: producer-block - tags: - phase: producer - admin_task: false - statements: - - name: s1 - optype: msg-send - # producer_name: {producer_name} - msg_key: "{mykey}" - msg_property: "{myprop}" - msg_value: | - { - "SensorID": "{sensor_id}", - "SensorType": "Temperature", - "ReadingTime": "{reading_time}", - "ReadingValue": {reading_value} - } - - - name: consumer-block - tags: - phase: consumer - admin_task: false - statements: - - name: s1 - optype: msg-consume -# topic_names: -# topics_pattern: - subscription_name: "mysub" -# subscription_type: -# consumer_name: - - - name: reader-block - tags: - phase: reader - admin_task: false - statements: - - name: s1 - optype: msg-read -# reader_name: - -# - websocket-producer: -# tags: -# type: websocket-produer -# statements: -# - websocket-producer-stuff: -# -# - managed-ledger: -# tags: -# type: managed-ledger -# statement: -# - managed-ledger-stuff: diff --git a/driver-pulsar/src/main/resources/activities/pulsar_client_kv.yaml b/driver-pulsar/src/main/resources/activities/pulsar_client_kv.yaml deleted file mode 100644 index a40592fca..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_client_kv.yaml +++ /dev/null @@ -1,96 +0,0 @@ -bindings: - # message key, property and value - mykey: NumberNameToString() - int_prop_val: ToString(); Prefix("IntProp_") - text_prop_val: AlphaNumericString(10); Prefix("TextProp_") - myvalue: NumberNameToString() #AlphaNumericString(20) - # tenant, namespace, and core topic name (without tenant and namespace) - tenant: Mod(100); Div(10L); ToString(); Prefix("tnt") - namespace: Mod(10); Div(5L); ToString(); Prefix("ns") - core_topic_name: Mod(5); ToString(); Prefix("t") - -# document level parameters that apply to all Pulsar client types: -params: - topic_uri: "persistent://{tenant}/{namespace}/{core_topic_name}" - async_api: "true" - -blocks: - - name: batch-producer-block - tags: - phase: batch-producer - admin_task: false - statements: - - name: s1 - optype: batch-msg-send-start - # For batch producer, "producer_name" should be associated with batch start - # batch_producer_name: {batch_producer_name} - ratio: 1 - - name: s2 - optype: batch-msg-send - msg_key: "{mykey}" - msg_property: | - { - "prop1": "{int_prop_val}", - "prop2": "{text_prop_val}}" - } - msg_value: "{myvalue}" - ratio: 100 - - name: s3 - optype: batch-msg-send-end - ratio: 1 - - - name: producer-block - tags: - phase: producer - admin_task: false - statements: - - name: s1 - optype: msg-send - # producer_name: {producer_name} -# msg_key: - msg_value: "{myvalue}" - - - name: consumer-block - tags: - phase: consumer - admin_task: false - statements: - - name: s1 - optype: msg-consume - subscription_name: "mysub" -# subscription_type: -# consumer_name: - - - name: reader-block - tags: - phase: reader - admin_task: false - statements: - - name: s1 - optype: msg-read -# reader_name: - - - name: multi-topic-consumer-block - tags: - phase: multi-topic-consumer - admin_task: false - statements: - - name: s1 - optype: msg-mt-consume -# topic_names: -# topics_pattern: - subscription_name: "mysub" -# subscription_type: -# consumer_name: - -# - websocket-producer: -# tags: -# type: websocket-producer -# statements: -# - websocket-producer-stuff: -# -# - managed-ledger: -# tags: -# type: managed-ledger -# statement: -# - managed-ledger-stuff: diff --git a/driver-pulsar/src/main/resources/activities/pulsar_client_sanity_e2e.yaml b/driver-pulsar/src/main/resources/activities/pulsar_client_sanity_e2e.yaml deleted file mode 100644 index 3cd88c831..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_client_sanity_e2e.yaml +++ /dev/null @@ -1,30 +0,0 @@ -bindings: - # message key, property and value - myprop1: AlphaNumericString(10); Prefix("PropVal_") - myvalue: NumberNameToString() #AlphaNumericString(20) - -# document level parameters that apply to all Pulsar client types: -params: - topic_uri: "persistent://public/default/sanity_e2e_2" - async_api: "true" - -blocks: - - name: e2e-msg-proc-block - tags: - phase: e2e-msg-proc - admin_task: false - statements: - - name: s1 - optype: ec2-msg-proc-send -# msg_key: - msg_property: | - { - "prop1": "{myprop1}" - } - msg_value: "{myvalue}" - ratio: 1 - - name: s2 - optype: ec2-msg-proc-consume - ratio: 1 - subscription_name: "mysub" -# subscription_type: diff --git a/driver-pulsar/src/main/resources/activities/pulsar_client_sanity_seqloss.yaml b/driver-pulsar/src/main/resources/activities/pulsar_client_sanity_seqloss.yaml deleted file mode 100644 index 8c1fd2c7f..000000000 --- a/driver-pulsar/src/main/resources/activities/pulsar_client_sanity_seqloss.yaml +++ /dev/null @@ -1,39 +0,0 @@ -bindings: - # message key, property and value - myprop1: AlphaNumericString(10) - myvalue: NumberNameToString() - -# document level parameters that apply to all Pulsar client types: -params: - topic_uri: "persistent://tnt0/ns0/sanity_seqloss12" - # Only applicable to producer and consumer - # - used for message ordering and message loss check - async_api: "true" - seq_tracking: "true" - -blocks: - - name: producer-block - tags: - phase: producer - admin_task: false - statements: - - name: s1 - optype: msg-send - #seqerr_simu: "out_of_order" - #seqerr_simu: "msg_loss" - #seqerr_simu: "msg_dup" - #seqerr_simu: "out_of_order, msg_loss" -# msg_key: -# msg_property: - msg_value: "{myvalue}" - - - name: consumer-block - tags: - phase: consumer - admin_task: false - statements: - - name: s1 - optype: msg-consume - subscription_name: "mysub" - subscription_type: "Shared" -# consumer_name: diff --git a/driver-pulsar/src/main/resources/design_revisit.md b/driver-pulsar/src/main/resources/design_revisit.md deleted file mode 100644 index 4070a39d3..000000000 --- a/driver-pulsar/src/main/resources/design_revisit.md +++ /dev/null @@ -1,89 +0,0 @@ -# TODO : Design Revisit -- Advanced Driver Features - -**NOTE**: The following text is based on the original multi-layer API -caching design which is not fully implemented at the moment. We need to -revisit the original design at some point in order to achieve maximum -testing flexibility. - -To summarize, the original caching design has the following key -requirements: - -* **Requirement 1**: Each NB Pulsar activity is able to launch and cache - multiple **client spaces** -* **Requirement 2**: Each client space can launch and cache multiple - Pulsar operators of the same type (producer, consumer, etc.) -* **Requirement 3**: The size of each Pulsar operator specific cached - space can be configurable. - -In the current implementation, only requirement 2 is implemented. - -* For requirement 1, the current implementation only supports one client - space per NB Pulsar activity -* For requirement 3, the cache space size is not configurable (no limit at - the moment) - -## Other Activity Parameters - -- **maxcached** - A default value to be applied to `max_clients`, - `max_producers`, `max_consumers`. - - default: `max_cached=100` -- **max_clients** - Clients cache size. This is the number of client - instances which are allowed to be cached in the NoSQLBench client - runtime. The clients cache automatically maintains a cache of unique - client instances internally. default: _maxcached_ -- **max_operators** - Producers/Consumers/Readers cache size (per client - instance). Limits the number of instances which are allowed to be cached - per client instance. default: _maxcached_ - -## API Caching - -This driver is tailored around the multi-tenancy and topic naming scheme -that is part of Apache Pulsar. Specifically, you can create an arbitrary -number of client instances, producers (per client), and consumers (per -client) depending on your testing requirements. - -Further, the topic URI is composed from the provided qualifiers of -`persistence`, `tenant`, `namespace`, and `topic`, or you can provide a -fully-composed value in the `persistence://tenant/namespace/topic` -form. - -### Instancing Controls - -Normative usage of the Apache Pulsar API follows a strictly enforced -binding of topics to producers and consumers. As well, clients may be -customized with different behavior for advanced testing scenarios. There -is a significant variety of messaging and concurrency schemes seen in -modern architectures. Thus, it is important that testing tools rise to the -occasion by letting users configure their testing runtimes to emulate -applications as they are found in practice. To this end, the NoSQLBench -driver for Apache Pulsar provides a set controls within its op template -format which allow for flexible yet intuitive instancing in the client -runtime. This is enabled directly by using nominative variables for -instance names where needed. When the instance names are not provided for -an operation, defaults are used to emulate a simple configuration. - -Since this is a new capability in a NoSQLBench driver, how it works is -explained below: - -When a pulsar cycles is executed, the operation is synthesized from the op -template fields as explained below under _Op Fields_. This happens in a -specific order: - -1. The client instance name is resolved. If a `client` field is provided, - this is taken as the client instance name. If not, it is set - to `default`. -2. The named client instance is fetched from the cache, or created and - cached if it does not yet exist. -3. The topic_uri is resolved. This is the value to be used with - `.topic(...)` calls in the API. The op fields below explain how to - control this value. -4. For _send_ operations, a producer is named and created if needed. By - default, the producer is named after the topic_uri above. You can - override this by providing a value for `producer`. -5. For _recv_ operations, a consumer is named and created if needed. By - default, the consumer is named after the topic_uri above. You can - override this by providing a value for `consumer`. - -The most important detail for understanding the instancing controls is -that clients, producers, and consumers are all named and cached in the -specific order above. diff --git a/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/TestYamlLoader.java b/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/TestYamlLoader.java deleted file mode 100644 index 21449034c..000000000 --- a/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/TestYamlLoader.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar; - -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; -import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.jupiter.api.Test; - -public class TestYamlLoader { - - private final static Logger logger = LogManager.getLogger(TestYamlLoader.class); - - @Test - public void loadAvroYaml() { - RawStmtsLoader sl = new RawStmtsLoader(); - RawStmtsDocList rsdl = sl.loadPath(logger, "activities/pulsar_client_avro.yaml"); - } -} diff --git a/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandlerTest.java b/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandlerTest.java deleted file mode 100644 index 322ad4da3..000000000 --- a/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/MessageSequenceNumberSendingHandlerTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import org.junit.jupiter.api.Test; - -class MessageSequenceNumberSendingHandlerTest { - MessageSequenceNumberSendingHandler sequenceNumberSendingHandler = new MessageSequenceNumberSendingHandler(); - - @Test - void shouldAddMonotonicSequence() { - for (long l = 1; l <= 100; l++) { - assertEquals(l, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - } - } - - @Test - void shouldInjectMessageLoss() { - assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - assertEquals(3L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.MsgLoss), 100)); - } - - @Test - void shouldInjectMessageDuplication() { - assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.MsgDup), 100)); - } - - @Test - void shouldInjectMessageOutOfOrder() { - assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - assertEquals(4L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.singleton(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.OutOfOrder), 100)); - assertEquals(2L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - assertEquals(3L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - assertEquals(5L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - assertEquals(6, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - } - - @Test - void shouldInjectOneOfTheSimulatedErrorsRandomly() { - Set allErrorTypes = new HashSet<>(Arrays.asList(PulsarActivityUtil.SEQ_ERROR_SIMU_TYPE.values())); - - assertEquals(1L, sequenceNumberSendingHandler.getNextSequenceNumber(Collections.emptySet())); - long previousSequenceNumber = 1L; - int outOfSequenceInjectionCounter = 0; - int messageDupCounter = 0; - int messageLossCounter = 0; - int successCounter = 0; - for (int i = 0; i < 1000; i++) { - long nextSequenceNumber = sequenceNumberSendingHandler.getNextSequenceNumber(allErrorTypes); - if (nextSequenceNumber >= previousSequenceNumber + 3) { - outOfSequenceInjectionCounter++; - } else if (nextSequenceNumber <= previousSequenceNumber) { - messageDupCounter++; - } else if (nextSequenceNumber >= previousSequenceNumber + 2) { - messageLossCounter++; - } else if (nextSequenceNumber == previousSequenceNumber + 1) { - successCounter++; - } - previousSequenceNumber = nextSequenceNumber; - } - assertTrue(outOfSequenceInjectionCounter > 0); - assertTrue(messageDupCounter > 0); - assertTrue(messageLossCounter > 0); - assertEquals(1000, outOfSequenceInjectionCounter + messageDupCounter + messageLossCounter + successCounter); - } - -} diff --git a/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTrackerTest.java b/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTrackerTest.java deleted file mode 100644 index 5258526d3..000000000 --- a/driver-pulsar/src/test/java/io/nosqlbench/driver/pulsar/ops/ReceivedMessageSequenceTrackerTest.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.pulsar.ops; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.codahale.metrics.Counter; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class ReceivedMessageSequenceTrackerTest { - Counter msgErrOutOfSeqCounter = new Counter(); - Counter msgErrDuplicateCounter = new Counter(); - Counter msgErrLossCounter = new Counter(); - ReceivedMessageSequenceTracker messageSequenceTracker = new ReceivedMessageSequenceTracker(msgErrOutOfSeqCounter, msgErrDuplicateCounter, msgErrLossCounter, 20, 20); - - @Test - void shouldCountersBeZeroWhenSequenceDoesntContainGaps() { - // when - for (long l = 0; l < 100L; l++) { - messageSequenceTracker.sequenceNumberReceived(l); - } - messageSequenceTracker.close(); - // then - assertEquals(0, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(0, msgErrLossCounter.getCount()); - } - - @ParameterizedTest - @ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L}) - void shouldDetectMsgLossWhenEverySecondMessageIsLost(long totalMessages) { - doShouldDetectMsgLoss(totalMessages, 2); - } - - @ParameterizedTest - @ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L}) - void shouldDetectMsgLossWhenEveryThirdMessageIsLost(long totalMessages) { - doShouldDetectMsgLoss(totalMessages, 3); - } - - @ParameterizedTest - @ValueSource(longs = {20L, 21L, 40L, 41L, 42L, 43L, 100L}) - void shouldDetectMsgLossWhenEvery21stMessageIsLost(long totalMessages) { - doShouldDetectMsgLoss(totalMessages, 21); - } - - private void doShouldDetectMsgLoss(long totalMessages, int looseEveryNthMessage) { - int messagesLost = 0; - // when - boolean lastMessageWasLost = false; - for (long l = 0; l < totalMessages; l++) { - if (l % looseEveryNthMessage == 1) { - messagesLost++; - lastMessageWasLost = true; - continue; - } else { - lastMessageWasLost = false; - } - messageSequenceTracker.sequenceNumberReceived(l); - } - if (lastMessageWasLost) { - messageSequenceTracker.sequenceNumberReceived(totalMessages); - } - messageSequenceTracker.close(); - // then - assertEquals(0, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(messagesLost, msgErrLossCounter.getCount()); - } - - @ParameterizedTest - @ValueSource(longs = {10L, 11L, 19L, 20L, 21L, 100L}) - void shouldDetectMsgDuplication(long totalMessages) { - int messagesDuplicated = 0; - // when - for (long l = 0; l < totalMessages; l++) { - if (l % 2 == 1) { - messagesDuplicated++; - messageSequenceTracker.sequenceNumberReceived(l); - } - messageSequenceTracker.sequenceNumberReceived(l); - } - if (totalMessages % 2 == 0) { - messageSequenceTracker.sequenceNumberReceived(totalMessages); - } - if (totalMessages < 2 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers()) { - messageSequenceTracker.close(); - } - - // then - assertEquals(0, msgErrOutOfSeqCounter.getCount()); - assertEquals(messagesDuplicated, msgErrDuplicateCounter.getCount()); - assertEquals(0, msgErrLossCounter.getCount()); - } - - @Test - void shouldDetectSingleMessageOutOfSequence() { - // when - for (long l = 0; l < 10L; l++) { - messageSequenceTracker.sequenceNumberReceived(l); - } - messageSequenceTracker.sequenceNumberReceived(10L); - messageSequenceTracker.sequenceNumberReceived(12L); - messageSequenceTracker.sequenceNumberReceived(11L); - for (long l = 13L; l < 100L; l++) { - messageSequenceTracker.sequenceNumberReceived(l); - } - - // then - assertEquals(1, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(0, msgErrLossCounter.getCount()); - } - - @Test - void shouldDetectMultipleMessagesOutOfSequence() { - // when - for (long l = 0; l < 10L; l++) { - messageSequenceTracker.sequenceNumberReceived(l); - } - messageSequenceTracker.sequenceNumberReceived(10L); - messageSequenceTracker.sequenceNumberReceived(14L); - messageSequenceTracker.sequenceNumberReceived(13L); - messageSequenceTracker.sequenceNumberReceived(11L); - messageSequenceTracker.sequenceNumberReceived(12L); - for (long l = 15L; l < 100L; l++) { - messageSequenceTracker.sequenceNumberReceived(l); - } - - // then - assertEquals(2, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(0, msgErrLossCounter.getCount()); - } - - @Test - void shouldDetectIndividualMessageLoss() { - // when - for (long l = 0; l < 100L; l++) { - if (l != 11L) { - messageSequenceTracker.sequenceNumberReceived(l); - } - } - messageSequenceTracker.close(); - - // then - assertEquals(0, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(1, msgErrLossCounter.getCount()); - } - - @Test - void shouldDetectGapAndMessageDuplication() { - // when - for (long l = 0; l < 100L; l++) { - if (l != 11L) { - messageSequenceTracker.sequenceNumberReceived(l); - } - if (l == 12L) { - messageSequenceTracker.sequenceNumberReceived(l); - } - } - messageSequenceTracker.close(); - - // then - assertEquals(0, msgErrOutOfSeqCounter.getCount()); - assertEquals(1, msgErrDuplicateCounter.getCount()); - assertEquals(1, msgErrLossCounter.getCount()); - } - - @Test - void shouldDetectGapAndMessageDuplicationTimes2() { - // when - for (long l = 0; l < 100L; l++) { - if (l != 11L) { - messageSequenceTracker.sequenceNumberReceived(l); - } - if (l == 12L) { - messageSequenceTracker.sequenceNumberReceived(l); - messageSequenceTracker.sequenceNumberReceived(l); - } - } - messageSequenceTracker.close(); - - // then - assertEquals(0, msgErrOutOfSeqCounter.getCount()); - assertEquals(2, msgErrDuplicateCounter.getCount()); - assertEquals(1, msgErrLossCounter.getCount()); - } - - - @Test - void shouldDetectDelayedOutOfOrderDelivery() { - // when - for (long l = 0; l < 5 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers(); l++) { - if (l != 10) { - messageSequenceTracker.sequenceNumberReceived(l); - } - if (l == messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers() * 2) { - messageSequenceTracker.sequenceNumberReceived(10); - } - } - messageSequenceTracker.close(); - - // then - assertEquals(1, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(0, msgErrLossCounter.getCount()); - } - - @Test - void shouldDetectDelayedOutOfOrderDeliveryOf2ConsecutiveSequenceNumbers() { - // when - for (long l = 0; l < 5 * messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers(); l++) { - if (l != 10 && l != 11) { - messageSequenceTracker.sequenceNumberReceived(l); - } - if (l == messageSequenceTracker.getMaxTrackOutOfOrderSequenceNumbers() * 2) { - messageSequenceTracker.sequenceNumberReceived(10); - messageSequenceTracker.sequenceNumberReceived(11); - } - } - messageSequenceTracker.close(); - - // then - assertEquals(2, msgErrOutOfSeqCounter.getCount()); - assertEquals(0, msgErrDuplicateCounter.getCount()); - assertEquals(0, msgErrLossCounter.getCount()); - } -} diff --git a/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java b/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java index bb8063827..050c51c57 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java +++ b/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java @@ -22,9 +22,11 @@ public class BundledMarkdownLoader { public static DocsBinder loadBundledMarkdown() { ServiceLoader loader = ServiceLoader.load(BundledMarkdownManifest.class); - Docs docs = new Docs(); + DocsBinder docs = new Docs(); for (BundledMarkdownManifest docPathInfos : loader) { - docs.merge(docPathInfos.getDocs()); + + DocsBinder docsBinder = docPathInfos.getDocs(); + docs = docs.merge(docsBinder); } return docs; diff --git a/nb-api/src/main/java/io/nosqlbench/api/markdown/aggregator/MutableMarkdown.java b/nb-api/src/main/java/io/nosqlbench/api/markdown/aggregator/MutableMarkdown.java index b3a6e3dab..f8c4feed9 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/markdown/aggregator/MutableMarkdown.java +++ b/nb-api/src/main/java/io/nosqlbench/api/markdown/aggregator/MutableMarkdown.java @@ -51,6 +51,11 @@ public class MutableMarkdown { throw new RuntimeException(e); } } + public MutableMarkdown(String rawMarkdown) { + this.path = null; + this.rawMarkdown = rawMarkdown; + parseStructure(rawMarkdown); + } private void parseStructure(String rawMarkdown) { AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor(); @@ -69,13 +74,19 @@ public class MutableMarkdown { } else if (node instanceof WhiteSpace) { } else if (node instanceof YamlFrontMatterBlock) { } else { - throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName()); + if(this.path != null) + throw new RuntimeException("The markdown file at '" + this.path.toString() + "' must have an initial heading as a title, before any other element, but found:" + node.getClass().getSimpleName()); + else + throw new RuntimeException("The markdown string provided must have an initial heading as a title, before any other element, but found: "+ node.getClass().getSimpleName()); } node=node.getNext(); } } if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) { - throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title."); + if(this.path != null) + throw new RuntimeException("The markdown file at '" + this.path.toString() + "' has no heading to use as a title."); + else + throw new RuntimeException("The markdown string provided has no heading to use as a title."); } } @@ -90,7 +101,10 @@ public class MutableMarkdown { if (end>=0) { return rawMarkdown.substring(end+4); } else { - throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary); + if(path != null) + throw new RuntimeException("Unable to find matching boundaries in " + path.toString() + ": " + boundary); + else + throw new RuntimeException("Unable to find matching boundaries in provided markdown: " + boundary); } } } diff --git a/nbr/pom.xml b/nbr/pom.xml index cc59c9caa..c623232b7 100644 --- a/nbr/pom.xml +++ b/nbr/pom.xml @@ -72,6 +72,7 @@ adapter-diag 4.17.32-SNAPSHOT + diff --git a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java similarity index 97% rename from nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java rename to nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java index 36dc7a0b3..06f93ec33 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java +++ b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledFrontmatterInjector.java @@ -39,7 +39,7 @@ public class BundledFrontmatterInjector implements BundledMarkdownProcessor { if (name.length()>i) { int ord = name.charAt(i) - 'a'; double addend = Math.pow(pow, i) * ord; - sum += addend; + sum += (int)addend; } else { break; } diff --git a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java similarity index 99% rename from nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java rename to nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java index 6cfb3b49d..a6c5ecc07 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java +++ b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java @@ -53,6 +53,7 @@ public class BundledMarkdownExporter implements BundledApp { String zipfile = options.valueOf(zipfileSpec); new BundledMarkdownZipExporter(new BundledFrontmatterInjector()).exportDocs(Path.of(zipfile)); + return 0; } } diff --git a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownProcessor.java b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownProcessor.java similarity index 100% rename from nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownProcessor.java rename to nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownProcessor.java diff --git a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java similarity index 65% rename from nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java rename to nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java index 3df95c740..5c2d752a2 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java +++ b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownZipExporter.java @@ -20,6 +20,7 @@ import io.nosqlbench.api.docsapi.BundledMarkdownLoader; import io.nosqlbench.api.docsapi.DocsBinder; import io.nosqlbench.api.docsapi.DocsNameSpace; import io.nosqlbench.api.markdown.aggregator.MutableMarkdown; +import io.nosqlbench.virtdata.userlibs.apps.docsapp.VirtDataGenDocsApp; import java.io.File; import java.io.IOException; @@ -29,7 +30,12 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.Date; import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.function.Function; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -51,12 +57,33 @@ public class BundledMarkdownZipExporter { zipstream.setMethod(ZipOutputStream.DEFLATED); zipstream.setLevel(9); - DocsBinder docsNameSpaces = BundledMarkdownLoader.loadBundledMarkdown(); + DocsBinder docsNameSpaces = BundledMarkdownLoader.loadBundledMarkdown(); //Loads the drivers under @Service Annotation + for (DocsNameSpace docs_ns : docsNameSpaces) { for (Path p : docs_ns) { - addEntry(p, p.getParent(), zipstream); + addEntry(p, p.getParent(), zipstream, docs_ns.getName() + "/"); } } + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future> future = executorService.submit(new VirtDataGenDocsApp(null)); + Map builderMap = future.get(); + executorService.shutdown(); + String bindingsPrefix ="bindings/"; + for(Map.Entry entry : builderMap.entrySet()) + { + String filename = entry.getKey(); + StringBuilder fileStringBuilder = entry.getValue(); + MutableMarkdown parsed = new MutableMarkdown(fileStringBuilder.toString()); + for (BundledMarkdownProcessor filter : this.filters) { + parsed = filter.apply(parsed); + } + ZipEntry zipEntry = new ZipEntry(bindingsPrefix + filename); + zipEntry.setTime(new Date().getTime()); + zipstream.putNextEntry(zipEntry); + zipstream.write(parsed.getComposedMarkdown().getBytes(StandardCharsets.UTF_8)); + zipstream.closeEntry(); + } zipstream.finish(); stream.close(); } catch (Exception e) { @@ -64,18 +91,17 @@ public class BundledMarkdownZipExporter { } } - private void addEntry(Path p, Path r, ZipOutputStream zos) throws IOException { + private void addEntry(Path p, Path r, ZipOutputStream zos, String prefix) throws IOException { String name = r.relativize(p).toString(); name = Files.isDirectory(p) ? (name.endsWith(File.separator) ? name : name + File.separator) : name; - - ZipEntry entry = new ZipEntry(name); + ZipEntry entry = new ZipEntry(prefix + name); if (Files.isDirectory(p)) { zos.putNextEntry(entry); DirectoryStream stream = Files.newDirectoryStream(p); for (Path path : stream) { - addEntry(path,r,zos); + addEntry(path,r,zos, prefix); } } else { entry.setTime(Files.getLastModifiedTime(p).toMillis()); diff --git a/nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java b/nbr/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java similarity index 100% rename from nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java rename to nbr/src/test/java/io/nosqlbench/docapi/BundledMarkdownExporterTest.java diff --git a/nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownTestManifest.java b/nbr/src/test/java/io/nosqlbench/docapi/BundledMarkdownTestManifest.java similarity index 100% rename from nb-api/src/test/java/io/nosqlbench/docapi/BundledMarkdownTestManifest.java rename to nbr/src/test/java/io/nosqlbench/docapi/BundledMarkdownTestManifest.java diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_string/EscapeJSON.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_string/EscapeJSON.java index 8e29cb549..7f73806fc 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_string/EscapeJSON.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_string/EscapeJSON.java @@ -18,6 +18,8 @@ package io.nosqlbench.virtdata.library.basics.shared.from_string; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import io.nosqlbench.virtdata.api.annotations.Categories; +import io.nosqlbench.virtdata.api.annotations.Category; import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; import org.apache.commons.text.StringEscapeUtils; @@ -38,6 +40,7 @@ import java.util.function.Function; * } */ @ThreadSafeMapper +@Categories({Category.conversion, Category.general}) public class EscapeJSON implements Function { Gson gson = new GsonBuilder().create(); diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java index 1ef438da7..d34fe2c0c 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java @@ -31,12 +31,11 @@ import java.util.Arrays; public class VirtDataMainApp implements BundledApp { private final static String APP_TESTMAPPER = "testmapper"; - private final static String APP_GENDOCS = "gendocs"; private final static String APP_DIAGNOSE = "diagnose"; - private final static String[] names = new String[]{APP_GENDOCS, APP_TESTMAPPER, APP_DIAGNOSE}; + private final static String[] names = new String[]{APP_TESTMAPPER, APP_DIAGNOSE}; public static boolean hasNamedApp(String appname) { - return (appname.equals(APP_TESTMAPPER) || appname.equals(APP_GENDOCS) || appname.equals(APP_DIAGNOSE)); + return (appname.equals(APP_TESTMAPPER) || appname.equals(APP_DIAGNOSE)); } public static void main(String[] args) { @@ -46,7 +45,7 @@ public class VirtDataMainApp implements BundledApp { @Override public int applyAsInt(String[] args) { if (args.length == 0) { - System.out.println("Usage: app (" + APP_TESTMAPPER + "|" + APP_GENDOCS + "|" + APP_DIAGNOSE +")"); + System.out.println("Usage: app (" + APP_TESTMAPPER +"|"+ APP_DIAGNOSE +")"); return 1; } @@ -58,8 +57,6 @@ public class VirtDataMainApp implements BundledApp { if (appSelection.equalsIgnoreCase(APP_TESTMAPPER)) { VirtDataCheckPerfApp.main(appArgs); - } else if (appSelection.equalsIgnoreCase(APP_GENDOCS)) { - VirtDataGenDocsApp.main(appArgs); } else if (appSelection.equalsIgnoreCase(APP_DIAGNOSE)) { VirtDataDiagnoseApp.main(appArgs); } else { diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/VirtDataGenDocsApp.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/VirtDataGenDocsApp.java index ca4c10058..dc872c7f4 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/VirtDataGenDocsApp.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/VirtDataGenDocsApp.java @@ -36,8 +36,9 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.concurrent.Callable; -public class VirtDataGenDocsApp implements Runnable { +public class VirtDataGenDocsApp implements Callable> { private final static Logger logger = LogManager.getLogger(VirtDataGenDocsApp.class); @@ -55,6 +56,7 @@ public class VirtDataGenDocsApp implements Runnable { private final String[] args; private final Map writers = new HashMap<>(); + Map stringBuilders = new HashMap(); private String baseFileName = BASE_FILENAME; private String categories = CATEGORIES_SPLIT; @@ -64,22 +66,21 @@ public class VirtDataGenDocsApp implements Runnable { private String basedir = ""; public static void main(String[] args) { - new VirtDataGenDocsApp(args).run(); + new VirtDataGenDocsApp(args).call(); } - public VirtDataGenDocsApp(String[] args) { - this.args = args; - } + public VirtDataGenDocsApp(String[] args) {this.args = args;} - public void run() { - LinkedList largs = new LinkedList<>(Arrays.asList(args)); + public Map call() { + + /*LinkedList largs = new LinkedList<>(Arrays.asList(args)); if (args.length > 0 && args[0].contains("help")) { System.out.println( "usage:\n" + "[basefile ] [basedir

] [categories combined|split] [format json|markdown] " + "[blurbsdirs [:...]]\n\n" ); - return; + return result; } while (largs.peekFirst() != null) { String argtype = largs.removeFirst(); @@ -112,12 +113,12 @@ public class VirtDataGenDocsApp implements Runnable { break; default: } - } + }*/ Optional docsinfo = loadAllDocs(); if (!docsinfo.isPresent()) { - return; + return stringBuilders; } try { @@ -131,38 +132,43 @@ public class VirtDataGenDocsApp implements Runnable { + (this.categories.equals(CATEGORIES_SPLIT) ? "_" + categoryName : "") + extension; - Writer writer = getWriterFor(filename); +// Writer writer = getWriterFor(filename); + StringBuilder builder = getBuilderFor(filename); + String name = filename; for (FDocFuncs docsForFuncName : docsForCatName) { if (format.equals(FORMAT_JSON)) { Gson gson = new GsonBuilder().setPrettyPrinting().create(); - writer.append(gson.toJson(docsForFuncName)); + builder.append(gson.toJson(docsForFuncName)); } else if (format.equals(FORMAT_MARKDOWN)) { String markdown = docsForFuncName.asMarkdown(); - writer.append(markdown); + builder.append(markdown); } } } - for (Writer writer : writers.values()) { + /*for (Writer writer : writers.values()) { writer.flush(); writer.close(); - } + }*/ } catch (Exception e) { + throw new RuntimeException(e); } + return stringBuilders; } - private Writer getWriterFor(String outputname) { - FileWriter fileWriter = null; - if (!writers.containsKey(outputname)) { + private StringBuilder getBuilderFor(String outputname) { +// FileWriter fileWriter = null; + StringBuilder builder = null; + if (!stringBuilders.containsKey(outputname)) { try { outputname = basedir.isEmpty() ? outputname : basedir + "/" + outputname; Path parent = Path.of(outputname).getParent(); if (parent != null) { Files.createDirectories(parent); } - fileWriter = new FileWriter(outputname, false); - writers.put(outputname, fileWriter); + builder = new StringBuilder(); + stringBuilders.put(outputname, builder); String[] blurbsdirs = blurbsDirs.split(":"); for (String blurbsdir : blurbsdirs) { @@ -173,7 +179,7 @@ public class VirtDataGenDocsApp implements Runnable { String blurb = Files.readString(blurbsFile, StandardCharsets.UTF_8); logger.debug("writing blurb to " + outputname); - fileWriter.append(blurb); + builder.append(blurb); } } } @@ -181,7 +187,7 @@ public class VirtDataGenDocsApp implements Runnable { throw new RuntimeException(e); } } - return writers.get(outputname); + return stringBuilders.get(outputname); } private Optional loadAllDocs() {