From c07fa49e041b533e187f4a445cb1c14c3a2ebd08 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 11 Oct 2024 15:11:58 -0500 Subject: [PATCH 01/27] allow prepared statements for testing --- .../cqld4/opdispensers/Cqld4PreparedStmtDispenser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java index acdfa6b65..4c0dedb73 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java @@ -47,8 +47,8 @@ public class Cqld4PreparedStmtDispenser extends Cqld4BaseOpDispenser { DriverAdapter adapter, LongFunction sessionFunc, ParsedOp op, ParsedTemplateString stmtTpl, RSProcessors processors) { super(adapter, sessionFunc, op); if (op.isDynamic("space")) { - throw new RuntimeException("Prepared statements and dynamic space values are not supported." + - " This would churn the prepared statement cache, defeating the purpose of prepared statements."); + throw new RuntimeException("Prepared statements and dynamic space values are not yet supported" + + " but are being implemented"); } this.processors = processors; this.stmtTpl = stmtTpl; From ddd39d923790b63e376ae1ea4b71dcfbb179ca03 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 11 Oct 2024 15:14:39 -0500 Subject: [PATCH 02/27] maven overrides --- .../adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java index 4c0dedb73..acf25c74c 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java @@ -47,7 +47,7 @@ public class Cqld4PreparedStmtDispenser extends Cqld4BaseOpDispenser { DriverAdapter adapter, LongFunction sessionFunc, ParsedOp op, ParsedTemplateString stmtTpl, RSProcessors processors) { super(adapter, sessionFunc, op); if (op.isDynamic("space")) { - throw new RuntimeException("Prepared statements and dynamic space values are not yet supported" + + throw new RuntimeException("Prepared statements and dynamic space values are not yet supported" + " but are being implemented"); } this.processors = processors; From 4fa93955c004a40b0b689162a8967b11d50809c4 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 11 Oct 2024 15:16:12 -0500 Subject: [PATCH 03/27] space metrics, heap metrics --- .../uniform/BaseDriverAdapter.java | 2 +- .../uniform/DriverSpaceCache.java | 15 ++++++- .../nb/api/config/standard/ConfigModel.java | 3 +- .../java/io/nosqlbench/engine/cli/NBCLI.java | 35 +-------------- .../core/lifecycle/session/NBSession.java | 45 +++++++++++++++++++ 5 files changed, 63 insertions(+), 37 deletions(-) diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java index 6fe41ffcf..c5494320b 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java @@ -131,7 +131,7 @@ public abstract class BaseDriverAdapter extends NBBaseComponent @Override public final synchronized DriverSpaceCache getSpaceCache() { if (spaceCache == null) { - spaceCache = new DriverSpaceCache<>(getSpaceInitializer(getConfiguration())); + spaceCache = new DriverSpaceCache<>(this, getSpaceInitializer(getConfiguration())); } return spaceCache; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java index a58007586..cb96c3456 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java @@ -16,6 +16,10 @@ package io.nosqlbench.adapters.api.activityimpl.uniform; +import io.nosqlbench.nb.api.components.core.NBBaseComponent; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory; + import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -42,14 +46,21 @@ import java.util.function.Function; * @param * The type which will represent the cache for a given type of adapter. */ -public class DriverSpaceCache { +public class DriverSpaceCache extends NBBaseComponent { private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); private final Function newSpaceFunction; - public DriverSpaceCache(Function newSpaceFunction) { + public DriverSpaceCache(NBComponent parent, Function newSpaceFunction) { + super(parent); this.newSpaceFunction = newSpaceFunction; + this.create().gauge( + "spaces", + () -> (double) this.getElements().size(), + MetricCategory.Internals, + "the number of spaces instantiated by this adapter instance" + ); } public S get(String name) { diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java index 27fbef95e..5e75a148e 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/ConfigModel.java @@ -159,8 +159,9 @@ public class ConfigModel implements NBConfigModel { @Override public NBConfiguration matchConfig(Map sharedConfig) { LinkedHashMap extracted = new LinkedHashMap<>(); + Map> namedParams = getNamedParams(); for (String providedCfgField : sharedConfig.keySet()) { - if (getNamedParams().containsKey(providedCfgField)) { + if (namedParams.containsKey(providedCfgField)) { extracted.put(providedCfgField, sharedConfig.get(providedCfgField)); } } diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index a7f8378aa..0c3720acd 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -317,8 +317,7 @@ public class NBCLI implements Function, NBLabeledElement { .pathname(resourceToCat).first(); final Content data = tocat.orElseThrow( - () -> new BasicError("Unable to find " + resourceToCat + - " in classpath to cat out")); + () -> new BasicError("Unable to find " + resourceToCat + " in classpath to cat out")); System.out.println(data.get()); NBCLI.logger.info(() -> "Dumped internal resource '" + data.asPath() + "' to stdout"); @@ -340,9 +339,7 @@ public class NBCLI implements Function, NBLabeledElement { .pathname(resourceToCopy).first(); final Content data = tocopy.orElseThrow( - () -> new BasicError( - "Unable to find " + resourceToCopy + - " in classpath to copy out") + () -> new BasicError("Unable to find " + resourceToCopy + " in classpath to copy out") ); final Path writeTo = Path.of(data.asPath().getFileName().toString()); @@ -381,7 +378,6 @@ public class NBCLI implements Function, NBLabeledElement { String topic = options.wantsTopicalHelpFor(); Optional infoFor = NBJavaCommandLoader.getInfoFor(topic); -// infoFor = infoFor.or(() -> MarkdownFinder.forHelpTopic(options.wantsTopicalHelpFor())); infoFor.ifPresent(info -> { System.out.print(info.getHelp()); @@ -409,29 +405,6 @@ public class NBCLI implements Function, NBLabeledElement { .build() ); -// if ((null != reportPromPushTo) || (null != reportGraphiteTo) || (null != options.wantsReportCsvTo())) { -// final MetricReporters reporters = MetricReporters.getInstance(); -// reporters.addRegistry("workloads", ActivityMetrics.getMetricRegistry()); -// -// if (null != reportPromPushTo) -// reporters.addPromPush(reportPromPushTo, options.wantsMetricsPrefix(), promPushConfig); -// if (null != reportGraphiteTo) reporters.addGraphite(reportGraphiteTo, options.wantsMetricsPrefix()); -// if (null != options.wantsReportCsvTo()) -// reporters.addCSVReporter(options.wantsReportCsvTo(), options.wantsMetricsPrefix()); -// if (options.wantsLoggedMetrics()) { -// reporters.addLogger(); -// } -// reporters.start(10, options.getReportInterval()); -// } -// -// if (options.getConsoleLogLevel().isGreaterOrEqualTo(NBLogLevel.WARN)) { -// options.setWantsStackTraces(true); -// NBCLI.logger.debug(() -> "enabling stack traces since log level is " + options.getConsoleLogLevel()); -// } - - // client machine metrics; TODO: modify pollInterval - - // intentionally not shown for warn-only NBCLI.logger.info(() -> "console logging level is " + options.getConsoleLogLevel()); Map props = Map.of( @@ -441,10 +414,6 @@ public class NBCLI implements Function, NBLabeledElement { "prompush_cache", "prompush_cache.txt", "heartbeat", String.valueOf(options.wantsHeartbeatIntervalMs()) ); - /** - * At this point, the command stream from the CLI should be handed into the session, and the session should - * marshal and transform it for any scenario invocations directly. - */ try ( NBSession session = new NBSession( diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java index 4bc14fd47..8cb40055a 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java @@ -31,6 +31,10 @@ import io.nosqlbench.nb.api.labels.NBLabeledElement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryManagerMXBean; +import java.lang.management.MemoryPoolMXBean; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -45,6 +49,7 @@ import java.util.function.Function; public class NBSession extends NBHeartbeatComponent implements Function, ExecutionResult>, NBTokenWords { private final static Logger logger = LogManager.getLogger(NBSession.class); // private final ClientSystemMetricChecker clientMetricChecker; + private MemoryMXBean mbean = ManagementFactory.getMemoryMXBean(); private final Map containers = new ConcurrentHashMap<>(); @@ -69,6 +74,46 @@ public class NBSession extends NBHeartbeatComponent implements Function (double) mbean.getHeapMemoryUsage().getUsed(), + MetricCategory.Internals, + "heap memory used for nb" + ); + create().gauge( + "on_heap_memory_max", + () -> (double) mbean.getHeapMemoryUsage().getMax(), + MetricCategory.Internals, + "heap memory max for nb" + ); + create().gauge( + "on_heap_memory_committed", + () -> (double) mbean.getHeapMemoryUsage().getCommitted(), + MetricCategory.Internals, + "heap memory committed for nb" + ); + + // off-heap + create().gauge( + "off_heap_memory_used", + () -> (double) mbean.getNonHeapMemoryUsage().getUsed(), + MetricCategory.Internals, + "off-heap memory used for nb" + ); + create().gauge( + "off_heap_memory_max", + () -> (double) mbean.getNonHeapMemoryUsage().getMax(), + MetricCategory.Internals, + "off-heap memory max for nb" + ); + create().gauge( + "off_heap_memory_committed", + () -> (double) mbean.getNonHeapMemoryUsage().getCommitted(), + MetricCategory.Internals, + "off-heap memory committed for nb" + ); + create().gauge( "session_time", () -> (double) System.nanoTime(), From 41b1296469f2befa2beedaad89315ce414bb5766 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 11 Oct 2024 15:21:01 -0500 Subject: [PATCH 04/27] rename driver space cache for upcoming changes --- .../io/nosqlbench/adapter/amqp/AmqpDriverAdapter.java | 4 ++-- .../java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java | 6 +++--- .../io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java | 4 ++-- .../adapter/cqld4/opmappers/Cqld4CoreOpMapper.java | 6 +++--- .../adapter/dynamodb/DynamoDBDriverAdapter.java | 4 ++-- .../io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java | 6 +++--- .../io/nosqlbench/adapter/http/HttpDriverAdapter.java | 4 ++-- .../io/nosqlbench/adapter/http/core/HttpOpMapper.java | 6 +++--- .../java/io/nosqlbench/adapter/http/HttpOpMapperTest.java | 4 ++-- .../io/nosqlbench/adapter/kafka/KafkaDriverAdapter.java | 4 ++-- .../java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java | 6 +++--- .../io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java | 6 +++--- .../java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java | 6 +++--- .../io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java | 4 ++-- .../java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java | 6 +++--- .../java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java | 4 ++-- .../main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java | 6 +++--- .../io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java | 4 ++-- .../java/io/nosqlbench/adapter/stdout/StdoutOpMapper.java | 6 +++--- .../adapter/tcpclient/TcpClientDriverAdapter.java | 4 ++-- .../nosqlbench/adapter/tcpclient/TcpClientOpMapper.java | 6 +++--- .../adapter/tcpserver/TcpServerDriverAdapter.java | 4 ++-- .../nosqlbench/adapter/tcpserver/TcpServerOpMapper.java | 6 +++--- .../api/activityimpl/uniform/BaseDriverAdapter.java | 8 ++++---- .../adapters/api/activityimpl/uniform/DriverAdapter.java | 4 ++-- ...{DriverSpaceCache.java => StringDriverSpaceCache.java} | 4 ++-- 26 files changed, 66 insertions(+), 66 deletions(-) rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/{DriverSpaceCache.java => StringDriverSpaceCache.java} (95%) diff --git a/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpDriverAdapter.java b/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpDriverAdapter.java index cba516a2e..d1c571464 100644 --- a/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpDriverAdapter.java +++ b/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpDriverAdapter.java @@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; import io.nosqlbench.nb.annotations.Service; @@ -41,7 +41,7 @@ public class AmqpDriverAdapter extends BaseDriverAdapter getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration adapterConfig = getConfiguration(); return new AmqpOpMapper(this, adapterConfig, spaceCache); } diff --git a/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java b/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java index 1847ea3c9..82ad285c4 100644 --- a/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java +++ b/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java @@ -23,7 +23,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; import org.apache.logging.log4j.LogManager; @@ -34,10 +34,10 @@ public class AmqpOpMapper implements OpMapper { private final static Logger logger = LogManager.getLogger(AmqpOpMapper.class); private final NBConfiguration cfg; - private final DriverSpaceCache spaceCache; + private final StringDriverSpaceCache spaceCache; private final DriverAdapter adapter; - public AmqpOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { + public AmqpOpMapper(DriverAdapter adapter, NBConfiguration cfg, StringDriverSpaceCache spaceCache) { this.cfg = cfg; this.spaceCache = spaceCache; this.adapter = adapter; diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java index 736e40ef5..b279067a2 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java @@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; @@ -46,7 +46,7 @@ public class Cqld4DriverAdapter extends BaseDriverAdapter { @Override public OpMapper getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration config = getConfiguration(); return new Cqld4CoreOpMapper(this, config, spaceCache); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java index b4ed66501..40f53a88d 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java @@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; @@ -35,11 +35,11 @@ public class Cqld4CoreOpMapper implements OpMapper { private final static Logger logger = LogManager.getLogger(Cqld4CoreOpMapper.class); - private final DriverSpaceCache cache; + private final StringDriverSpaceCache cache; private final NBConfiguration cfg; private final DriverAdapter adapter; - public Cqld4CoreOpMapper(DriverAdapter adapter, NBConfiguration config, DriverSpaceCache cache) { + public Cqld4CoreOpMapper(DriverAdapter adapter, NBConfiguration config, StringDriverSpaceCache cache) { this.cfg = config; this.cache = cache; this.adapter = adapter; diff --git a/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBDriverAdapter.java b/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBDriverAdapter.java index 0bd7802fe..e0d75ce28 100644 --- a/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBDriverAdapter.java +++ b/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBDriverAdapter.java @@ -20,7 +20,7 @@ import io.nosqlbench.adapter.dynamodb.optypes.DynamoDBOp; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; import io.nosqlbench.nb.annotations.Maturity; @@ -39,7 +39,7 @@ public class DynamoDBDriverAdapter extends BaseDriverAdapter getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration adapterConfig = getConfiguration(); return new DynamoDBOpMapper(this, adapterConfig, spaceCache); } diff --git a/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java b/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java index faf3a3816..0e5f50a51 100644 --- a/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java +++ b/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java @@ -22,7 +22,7 @@ import io.nosqlbench.adapter.dynamodb.optypes.DynamoDBOp; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; import io.nosqlbench.nb.api.config.standard.NBConfiguration; @@ -30,10 +30,10 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; public class DynamoDBOpMapper implements OpMapper { private final NBConfiguration cfg; - private final DriverSpaceCache cache; + private final StringDriverSpaceCache cache; private final DriverAdapter adapter; - public DynamoDBOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache cache) { + public DynamoDBOpMapper(DriverAdapter adapter, NBConfiguration cfg, StringDriverSpaceCache cache) { this.cfg = cfg; this.cache = cache; this.adapter = adapter; diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java index 6da85376e..62f1ebdf7 100644 --- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java +++ b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java @@ -23,7 +23,7 @@ import io.nosqlbench.nb.api.config.standard.Param; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory; import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram; import io.nosqlbench.nb.api.labels.NBLabels; @@ -55,7 +55,7 @@ public class HttpDriverAdapter extends BaseDriverAdapter { @Override public OpMapper getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration config = getConfiguration(); return new HttpOpMapper(this, config, spaceCache); } diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java index 1916984e4..c19c02c0f 100644 --- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java +++ b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java @@ -19,7 +19,7 @@ package io.nosqlbench.adapter.http.core; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.nb.api.config.standard.NBConfiguration; @@ -28,10 +28,10 @@ import java.util.function.LongFunction; public class HttpOpMapper implements OpMapper { private final NBConfiguration cfg; - private final DriverSpaceCache spaceCache; + private final StringDriverSpaceCache spaceCache; private final DriverAdapter adapter; - public HttpOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { + public HttpOpMapper(DriverAdapter adapter, NBConfiguration cfg, StringDriverSpaceCache spaceCache) { this.cfg = cfg; this.spaceCache = spaceCache; this.adapter = adapter; diff --git a/nb-adapters/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java b/nb-adapters/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java index e5e009c6f..fa4969cea 100644 --- a/nb-adapters/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java +++ b/nb-adapters/adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java @@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityconfig.OpsLoader; import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate; import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat; import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.nb.api.labels.NBLabels; import org.apache.logging.log4j.LogManager; @@ -49,7 +49,7 @@ public class HttpOpMapperTest { HttpOpMapperTest.cfg = HttpSpace.getConfigModel().apply(Map.of()); HttpOpMapperTest.adapter = new HttpDriverAdapter(new TestComponent("parent","parent"), NBLabels.forKV()); HttpOpMapperTest.adapter.applyConfig(HttpOpMapperTest.cfg); - final DriverSpaceCache cache = HttpOpMapperTest.adapter.getSpaceCache(); + final StringDriverSpaceCache cache = HttpOpMapperTest.adapter.getSpaceCache(); HttpOpMapperTest.mapper = new HttpOpMapper(HttpOpMapperTest.adapter, HttpOpMapperTest.cfg, cache); } diff --git a/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaDriverAdapter.java b/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaDriverAdapter.java index 66afb8291..a29924d9e 100644 --- a/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaDriverAdapter.java +++ b/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaDriverAdapter.java @@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; import io.nosqlbench.nb.annotations.Service; @@ -41,7 +41,7 @@ public class KafkaDriverAdapter extends BaseDriverAdapter { @Override public OpMapper getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration adapterConfig = getConfiguration(); return new KafkaOpMapper(this, adapterConfig, spaceCache); } diff --git a/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java b/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java index 2ca132464..12a9b5cbe 100644 --- a/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java +++ b/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java @@ -23,7 +23,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; import org.apache.logging.log4j.LogManager; @@ -34,10 +34,10 @@ public class KafkaOpMapper implements OpMapper { private final static Logger logger = LogManager.getLogger(KafkaOpMapper.class); private final NBConfiguration cfg; - private final DriverSpaceCache spaceCache; + private final StringDriverSpaceCache spaceCache; private final DriverAdapter adapter; - public KafkaOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { + public KafkaOpMapper(DriverAdapter adapter, NBConfiguration cfg, StringDriverSpaceCache spaceCache) { this.cfg = cfg; this.spaceCache = spaceCache; this.adapter = adapter; diff --git a/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java b/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java index 2600bb0fb..4eecb33c6 100644 --- a/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java +++ b/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java @@ -21,7 +21,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; @@ -36,10 +36,10 @@ public class MongoOpMapper implements OpMapper { private final MongodbDriverAdapter adapter; private final NBConfiguration configuration; - private final DriverSpaceCache spaceCache; + private final StringDriverSpaceCache spaceCache; public MongoOpMapper(MongodbDriverAdapter adapter, NBConfiguration cfg, - DriverSpaceCache spaceCache) { + StringDriverSpaceCache spaceCache) { this.configuration = cfg; this.spaceCache = spaceCache; this.adapter = adapter; diff --git a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java index 1f0691b61..46b680f81 100644 --- a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java +++ b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java @@ -21,7 +21,7 @@ import io.nosqlbench.adapter.neo4j.ops.Neo4JBaseOp; import io.nosqlbench.adapter.neo4j.types.Neo4JOpType; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; @@ -29,10 +29,10 @@ import java.util.function.LongFunction; public class Neo4JOpMapper implements OpMapper { - private final DriverSpaceCache cache; + private final StringDriverSpaceCache cache; private final Neo4JDriverAdapter adapter; - public Neo4JOpMapper(Neo4JDriverAdapter adapter, DriverSpaceCache cache) { + public Neo4JOpMapper(Neo4JDriverAdapter adapter, StringDriverSpaceCache cache) { this.adapter = adapter; this.cache = cache; } diff --git a/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java b/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java index 15df97bd1..fa5938e83 100644 --- a/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java +++ b/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java @@ -20,7 +20,7 @@ import io.nosqlbench.adapter.pulsar.ops.PulsarOp; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; import io.nosqlbench.nb.annotations.Service; @@ -42,7 +42,7 @@ public class PulsarDriverAdapter extends BaseDriverAdapter getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration adapterConfig = getConfiguration(); return new PulsarOpMapper(this, adapterConfig, spaceCache); } diff --git a/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java index 623e22332..f1ebd9f13 100644 --- a/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java +++ b/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; import org.apache.logging.log4j.LogManager; @@ -33,10 +33,10 @@ public class PulsarOpMapper implements OpMapper { private final static Logger logger = LogManager.getLogger(PulsarOpMapper.class); private final NBConfiguration cfg; - private final DriverSpaceCache spaceCache; + private final StringDriverSpaceCache spaceCache; private final DriverAdapter adapter; - public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { + public PulsarOpMapper(DriverAdapter adapter, NBConfiguration cfg, StringDriverSpaceCache spaceCache) { this.cfg = cfg; this.spaceCache = spaceCache; this.adapter = adapter; diff --git a/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java b/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java index f995bc21e..519ffbb17 100644 --- a/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java +++ b/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java @@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; import io.nosqlbench.nb.annotations.Service; @@ -41,7 +41,7 @@ public class S4JDriverAdapter extends BaseDriverAdapter { @Override public OpMapper getOpMapper() { - DriverSpaceCache spaceCache = getSpaceCache(); + StringDriverSpaceCache spaceCache = getSpaceCache(); NBConfiguration adapterConfig = getConfiguration(); return new S4JOpMapper(this, adapterConfig, spaceCache); } diff --git a/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java b/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java index 6ad721d45..652908b2a 100644 --- a/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java +++ b/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java @@ -23,7 +23,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; import org.apache.logging.log4j.LogManager; @@ -34,10 +34,10 @@ public class S4JOpMapper implements OpMapper { private final static Logger logger = LogManager.getLogger(S4JOpMapper.class); private final NBConfiguration cfg; - private final DriverSpaceCache spaceCache; + private final StringDriverSpaceCache spaceCache; private final DriverAdapter adapter; - public S4JOpMapper(DriverAdapter adapter, NBConfiguration cfg, DriverSpaceCache spaceCache) { + public S4JOpMapper(DriverAdapter adapter, NBConfiguration cfg, StringDriverSpaceCache spaceCache) { this.cfg = cfg; this.spaceCache = spaceCache; this.adapter = adapter; diff --git a/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java b/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java index 4f5180f05..09060063e 100644 --- a/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java +++ b/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java @@ -22,7 +22,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; @@ -48,7 +48,7 @@ public class StdoutDriverAdapter extends BaseDriverAdapter getOpMapper() { - DriverSpaceCache ctxCache = getSpaceCache(); + StringDriverSpaceCache ctxCache = getSpaceCache(); return new StdoutOpMapper(this, ctxCache); } diff --git a/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutOpMapper.java b/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutOpMapper.java index ef9eb50ad..e9d2849b4 100644 --- a/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutOpMapper.java +++ b/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutOpMapper.java @@ -19,17 +19,17 @@ package io.nosqlbench.adapter.stdout; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; public class StdoutOpMapper implements OpMapper { - private final DriverSpaceCache ctxcache; + private final StringDriverSpaceCache ctxcache; private final DriverAdapter adapter; - public StdoutOpMapper(DriverAdapter adapter, DriverSpaceCache ctxcache) { + public StdoutOpMapper(DriverAdapter adapter, StringDriverSpaceCache ctxcache) { this.ctxcache = ctxcache; this.adapter = adapter; } diff --git a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java index 80d854041..efffc6aff 100644 --- a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java +++ b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java @@ -25,7 +25,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; @@ -49,7 +49,7 @@ public class TcpClientDriverAdapter extends BaseDriverAdapter getOpMapper() { - DriverSpaceCache ctxCache = getSpaceCache(); + StringDriverSpaceCache ctxCache = getSpaceCache(); return new TcpClientOpMapper(this,ctxCache); } diff --git a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java index dfd45aa6f..b9001ad97 100644 --- a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java +++ b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java @@ -18,18 +18,18 @@ package io.nosqlbench.adapter.tcpclient; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; public class TcpClientOpMapper implements OpMapper { - private final DriverSpaceCache ctxcache; + private final StringDriverSpaceCache ctxcache; private final TcpClientDriverAdapter adapter; - public TcpClientOpMapper(TcpClientDriverAdapter adapter, DriverSpaceCache ctxcache) { + public TcpClientOpMapper(TcpClientDriverAdapter adapter, StringDriverSpaceCache ctxcache) { this.ctxcache = ctxcache; this.adapter = adapter; } diff --git a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java index f65d6c54e..3bfa0d7d9 100644 --- a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java +++ b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java @@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; @@ -48,7 +48,7 @@ public class TcpServerDriverAdapter extends BaseDriverAdapter getOpMapper() { - DriverSpaceCache ctxCache = getSpaceCache(); + StringDriverSpaceCache ctxCache = getSpaceCache(); return new TcpServerOpMapper(this,ctxCache); } diff --git a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java index 7d894288b..c7d600948 100644 --- a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java +++ b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java @@ -18,18 +18,18 @@ package io.nosqlbench.adapter.tcpserver; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; -import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.adapters.api.activityimpl.uniform.StringDriverSpaceCache; import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; public class TcpServerOpMapper implements OpMapper { - private final DriverSpaceCache ctxcache; + private final StringDriverSpaceCache ctxcache; private final TcpServerDriverAdapter adapter; - public TcpServerOpMapper(TcpServerDriverAdapter adapter, DriverSpaceCache ctxcache) { + public TcpServerOpMapper(TcpServerDriverAdapter adapter, StringDriverSpaceCache ctxcache) { this.ctxcache = ctxcache; this.adapter = adapter; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java index c5494320b..7976e6ff5 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java @@ -37,7 +37,7 @@ import java.util.stream.Collectors; public abstract class BaseDriverAdapter extends NBBaseComponent implements DriverAdapter, NBConfigurable, NBReconfigurable { private final static Logger logger = LogManager.getLogger("ADAPTER"); - private DriverSpaceCache spaceCache; + private StringDriverSpaceCache spaceCache; private NBConfiguration cfg; private LongFunction spaceF; @@ -129,9 +129,9 @@ public abstract class BaseDriverAdapter extends NBBaseComponent } @Override - public final synchronized DriverSpaceCache getSpaceCache() { + public final synchronized StringDriverSpaceCache getSpaceCache() { if (spaceCache == null) { - spaceCache = new DriverSpaceCache<>(this, getSpaceInitializer(getConfiguration())); + spaceCache = new StringDriverSpaceCache<>(this, getSpaceInitializer(getConfiguration())); } return spaceCache; } @@ -193,7 +193,7 @@ public abstract class BaseDriverAdapter extends NBBaseComponent @Override public LongFunction getSpaceFunc(ParsedOp pop) { LongFunction spaceNameF = pop.getAsFunctionOr("space", "default"); - DriverSpaceCache cache = getSpaceCache(); + StringDriverSpaceCache cache = getSpaceCache(); return l -> getSpaceCache().get(spaceNameF.apply(l)); } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java index 9f3be1fd5..f35f0d81b 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java @@ -137,7 +137,7 @@ public interface DriverAdapter extends NBComponent * things needed by operations, or things needed during the * construction of operations. * - * See {@link DriverSpaceCache} for details on when and how to use this function. + * See {@link StringDriverSpaceCache} for details on when and how to use this function. * *

During Adapter Initialization, Op Mapping, Op Synthesis, or Op Execution, * you may need access to the objects in (the or a) space cache. You can build the @@ -146,7 +146,7 @@ public interface DriverAdapter extends NBComponent * * @return A cache of named objects */ - DriverSpaceCache getSpaceCache(); + StringDriverSpaceCache getSpaceCache(); /** * This method allows each driver adapter to create named state which is automatically diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/StringDriverSpaceCache.java similarity index 95% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/StringDriverSpaceCache.java index cb96c3456..cbafd911f 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverSpaceCache.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/StringDriverSpaceCache.java @@ -46,13 +46,13 @@ import java.util.function.Function; * @param * The type which will represent the cache for a given type of adapter. */ -public class DriverSpaceCache extends NBBaseComponent { +public class StringDriverSpaceCache extends NBBaseComponent { private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); private final Function newSpaceFunction; - public DriverSpaceCache(NBComponent parent, Function newSpaceFunction) { + public StringDriverSpaceCache(NBComponent parent, Function newSpaceFunction) { super(parent); this.newSpaceFunction = newSpaceFunction; this.create().gauge( From 22ff75aa9871acd0f1fbd410a2eb9ac1191c970a Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Oct 2024 13:39:26 -0500 Subject: [PATCH 05/27] implement concurrent index cache for spaces --- .../uniform/ConcurrentIndexCache.java | 137 ++++++++++++++++++ .../uniform/ConcurrentSpaceCache.java | 28 ++++ .../activityimpl/uniform/DriverAdapter.java | 2 +- .../uniform/ConcurrentIndexCacheTest.java | 34 +++++ 4 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCache.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentSpaceCache.java create mode 100644 nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCacheTest.java diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCache.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCache.java new file mode 100644 index 000000000..93a70efa6 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCache.java @@ -0,0 +1,137 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.nb.api.errors.OpConfigError; +import org.jetbrains.annotations.NotNull; + +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.function.LongFunction; + +public class ConcurrentIndexCache implements Iterable { + private final AtomicReference> cacheRef; + private static final int GROWTH_FACTOR = 2; + private final LongFunction valueLoader; + private final BitSet active = new BitSet(); + + // Constructor with initial capacity + public ConcurrentIndexCache(LongFunction valueLoader) { + this.valueLoader = valueLoader; + this.cacheRef = new AtomicReference<>(new AtomicReferenceArray<>(1)); + } + + // Get or compute value if absent, using a valueLoader function + public T get(long longkey) { + if (longkey > Integer.MAX_VALUE) { + throw new OpConfigError("space index must be between 0 and " + (Integer.MAX_VALUE - 1) + " inclusive"); + } + int key = (int)longkey; + + AtomicReferenceArray currentCache = cacheRef.get(); + + if (key >= currentCache.length()) { + resize(key); + currentCache = cacheRef.get(); // Get the updated array after resizing + } + + T value = currentCache.get(key); + if (value == null) { + T newValue; + synchronized (valueLoader) { // limit construction concurrency to 1 for now to avoid wasteful races + newValue = valueLoader.apply(key); + } + // Atomically set the value if it's still null (compare-and-set) + if (currentCache.compareAndSet(key, null, newValue)) { + active.set(key); + return newValue; + } else { + // Another thread might have set the value, so return the existing one + return currentCache.get(key); + } + } + return value; + } + + // Method to resize the array if key exceeds current capacity + private synchronized void resize(int key) { + AtomicReferenceArray currentCache = cacheRef.get(); + if (key < currentCache.length()) { + return; // Double-check locking to avoid multiple resizes + } + + // Calculate new size (at least as large as key + 1) + int newCapacity = Math.max(currentCache.length() * GROWTH_FACTOR, key + 1); + AtomicReferenceArray newCache = new AtomicReferenceArray<>(newCapacity); + + // Copy elements from old cache to new cache + for (int i = 0; i < currentCache.length(); i++) { + newCache.set(i, currentCache.get(i)); + } + + // Atomically update the cache reference + cacheRef.set(newCache); + } + + // Optional: Method to remove an entry + public T remove(int key) { + AtomicReferenceArray currentCache = cacheRef.get(); + if (key >= currentCache.length()) { + return null; // Key is out of bounds + } + + T oldValue = currentCache.get(key); + currentCache.set(key, null); // Set the slot to null (safe for garbage collection) + active.clear(key); + return oldValue; + } + + // Optional: Method to clear the entire cache + public void clear() { + cacheRef.set(new AtomicReferenceArray<>(1)); + } + + @Override + public @NotNull Iterator iterator() { + return new ElementIterator<>(this); + } + + public static final class ElementIterator implements @NotNull Iterator { + + private final PrimitiveIterator.OfInt iterator; + private final ConcurrentIndexCache indexCache; + + public ElementIterator(ConcurrentIndexCache ts) { + this.indexCache = ts; + iterator = ts.active.stream().iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public T next() { + int index = this.iterator.nextInt(); + return indexCache.get(index); + } + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentSpaceCache.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentSpaceCache.java new file mode 100644 index 000000000..de3582a1a --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentSpaceCache.java @@ -0,0 +1,28 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import java.util.function.LongFunction; + +public class ConcurrentSpaceCache extends ConcurrentIndexCache { + public ConcurrentSpaceCache(LongFunction valueLoader) { + super(valueLoader); + } + +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java index f35f0d81b..a05adeb14 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java @@ -146,7 +146,7 @@ public interface DriverAdapter extends NBComponent * * @return A cache of named objects */ - StringDriverSpaceCache getSpaceCache(); + ConcurrentSpaceCache getSpaceCache(); /** * This method allows each driver adapter to create named state which is automatically diff --git a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCacheTest.java b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCacheTest.java new file mode 100644 index 000000000..4d313d6d8 --- /dev/null +++ b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/activityimpl/uniform/ConcurrentIndexCacheTest.java @@ -0,0 +1,34 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +class ConcurrentIndexCacheTest { + @Test + public void testBasicCache() { + ConcurrentIndexCache sc = new ConcurrentIndexCache<>(l -> String.valueOf(l)); + String s = sc.get(300); + assertThat(s).isEqualTo("300"); + } + +} From 7b8e9145d73c2e055e52b2778c6c970d97fc3617 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Oct 2024 13:39:37 -0500 Subject: [PATCH 06/27] formalize space type in generics formalize space --- .../adapter/diag/DriverAdapterLoader.java | 3 +- .../api/activityimpl/BaseOpDispenser.java | 15 ++++---- .../api/activityimpl/OpDispenser.java | 9 +++-- .../adapters/api/activityimpl/OpMapper.java | 21 +++++++---- .../docs/BundledDriverAdapterDocs.java | 6 ++- .../uniform/BaseDriverAdapter.java | 19 +++++----- .../api/activityimpl/uniform/BaseSpace.java | 36 ++++++++++++++++++ .../activityimpl/uniform/DriverAdapter.java | 7 ++-- .../api/activityimpl/uniform/Space.java | 37 +++++++++++++++++++ .../api/activityimpl/SimpleActivity.java | 12 +++--- .../uniform/StandardActivity.java | 24 ++++++------ 11 files changed, 141 insertions(+), 48 deletions(-) create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapter/diag/DriverAdapterLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapter/diag/DriverAdapterLoader.java index f16dd88f8..c243b4e66 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapter/diag/DriverAdapterLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapter/diag/DriverAdapterLoader.java @@ -19,10 +19,11 @@ package io.nosqlbench.adapter.diag; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.nb.api.components.core.NBComponent; public interface DriverAdapterLoader { - public DriverAdapter load(NBComponent parent, NBLabels childLabels); + public DriverAdapter load(NBComponent parent, NBLabels childLabels); } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java index 8a7f45d2a..d100b98e8 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapters.api.activityimpl; import com.codahale.metrics.Timer; import groovy.lang.Binding; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.evalctx.*; import io.nosqlbench.adapters.api.metrics.ThreadLocalNamedTimers; @@ -42,10 +43,10 @@ import java.util.concurrent.TimeUnit; * Some details are tracked per op template, which aligns to the life-cycle of the op dispenser. * Thus, each op dispenser is where the stats for all related operations are kept. * - * @param + * @param * The type of operation */ -public abstract class BaseOpDispenser extends NBBaseComponent implements OpDispenser{ +public abstract class BaseOpDispenser extends NBBaseComponent implements OpDispenser{ protected final static Logger logger = LogManager.getLogger(BaseOpDispenser.class); public static final String VERIFIER = "verifier"; public static final String VERIFIER_INIT = "verifier-init"; @@ -55,7 +56,7 @@ public abstract class BaseOpDispenser extends NBBaseComponent i public static final String STOP_TIMERS = "stop-timers"; private final String opName; - protected final DriverAdapter adapter; + protected final DriverAdapter adapter; private final NBLabels labels; public final Timer verifierTimer; private boolean instrument; @@ -77,7 +78,7 @@ public abstract class BaseOpDispenser extends NBBaseComponent i private final CycleFunction _verifier; private final ThreadLocal> tlVerifier; - protected BaseOpDispenser(final DriverAdapter adapter, final ParsedOp op) { + protected BaseOpDispenser(final DriverAdapter adapter, final ParsedOp op) { super(adapter); opName = op.getName(); this.adapter = adapter; @@ -177,7 +178,7 @@ public abstract class BaseOpDispenser extends NBBaseComponent i return this.opName; } - public DriverAdapter getAdapter() { + public DriverAdapter getAdapter() { return this.adapter; } @@ -227,8 +228,8 @@ public abstract class BaseOpDispenser extends NBBaseComponent i } @Override - public final T apply(long value) { - T op = getOp(value); + public final OP apply(long value) { + OP op = getOp(value); return op; } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java index 57ff77e0e..9a5a823fd 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapters.api.activityimpl; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.evalctx.CycleFunction; import java.util.function.LongFunction; @@ -64,11 +65,11 @@ import java.util.function.LongFunction; * are easy to understand at the mapping level ({@link OpMapper}), * and streamlined for fast execution at the synthesis level ({@link OpDispenser}). * - * @param The parameter type of the actual operation which will be used + * @param The parameter type of the actual operation which will be used * to hold all the details for executing an operation, - * something that implements {@link Runnable}. + * something that implements {@link Op}. */ -public interface OpDispenser extends LongFunction, OpResultTracker { +public interface OpDispenser extends LongFunction, OpResultTracker { /** * The apply method in an op dispenser should do all the work of @@ -82,7 +83,7 @@ public interface OpDispenser extends LongFunction, OpResultTracker { * @return an executable operation */ - T getOp(long value); + OP getOp(long value); CycleFunction getVerifier(); diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java index 066c5fa8f..8bd2acc74 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java @@ -17,17 +17,21 @@ package io.nosqlbench.adapters.api.activityimpl; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.templating.ParsedOp; +import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.function.LongFunction; /** *

*

Synopsis

* An OpMapper is responsible for converting parsed op templates * into dispensers of operations based on the intention of the user. - * + *

* Op Templates as expressed as a set of field values, some literal, and * some dynamic, to be generated based on a specific cycle value. *

@@ -79,16 +83,19 @@ import java.util.function.Function; * to hold all the details for executing an operation, * generally something that implements {@link Runnable}. */ -public interface OpMapper extends Function> { +public interface OpMapper + extends BiFunction, OpDispenser> { /** * Interrogate the parsed command, and provide a new * - * @param op The {@link ParsedOp} which is the parsed version of the user-provided op template. - * This contains all the fields provided by the user, as well as explicit knowledge of - * which ones are static and dynamic. + * @param op + * The {@link ParsedOp} which is the parsed version of the user-provided op template. + * This contains all the fields provided by the user, as well as explicit knowledge of + * which ones are static and dynamic. + * @param spaceInitF * @return An OpDispenser which can be used to synthesize real operations. */ - @Override - OpDispenser apply(ParsedOp op); + OpDispenser apply(ParsedOp op, LongFunction spaceInitF); + } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/docs/BundledDriverAdapterDocs.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/docs/BundledDriverAdapterDocs.java index 457e950cd..c247f9fbf 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/docs/BundledDriverAdapterDocs.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/docs/BundledDriverAdapterDocs.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapters.api.activityimpl.docs; import io.nosqlbench.adapter.diag.DriverAdapterLoader; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.nb.api.docsapi.BundledMarkdownManifest; import io.nosqlbench.nb.api.docsapi.Docs; @@ -40,7 +41,10 @@ public class BundledDriverAdapterDocs implements BundledMarkdownManifest { List> namedProviders = loader.getNamedProviders(); for (SimpleServiceLoader.Component namedProvider : namedProviders) { DriverAdapterLoader driverAdapterLoader = namedProvider.provider.get(); - DriverAdapter driverAdapter = driverAdapterLoader.load(NBComponent.EMPTY_COMPONENT, NBLabels.forKV()); + DriverAdapter driverAdapter = driverAdapterLoader.load( + NBComponent.EMPTY_COMPONENT, + NBLabels.forKV() + ); DocsBinder bundledDocs = driverAdapter.getBundledDocs(); docs = docs.merge(bundledDocs); } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java index 7976e6ff5..20654c2b1 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java @@ -32,12 +32,13 @@ import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.function.LongFunction; +import java.util.function.LongToIntFunction; import java.util.stream.Collectors; -public abstract class BaseDriverAdapter extends NBBaseComponent implements DriverAdapter, NBConfigurable, NBReconfigurable { +public abstract class BaseDriverAdapter extends NBBaseComponent implements DriverAdapter, NBConfigurable, NBReconfigurable { private final static Logger logger = LogManager.getLogger("ADAPTER"); - private StringDriverSpaceCache spaceCache; + private ConcurrentSpaceCache spaceCache; private NBConfiguration cfg; private LongFunction spaceF; @@ -62,7 +63,7 @@ public abstract class BaseDriverAdapter extends NBBaseComponent mappers.addAll(stmtRemappers); mappers.addAll(getOpFieldRemappers()); - if (mappers.size() == 0) { + if (mappers.isEmpty()) { return (i) -> i; } @@ -121,7 +122,7 @@ public abstract class BaseDriverAdapter extends NBBaseComponent *

Provide a list of field remappers which operate on arbitrary fields. * Each function is applied to the op template fields.

* - * @return + * @return op field remappers, an empty list by default */ @Override public List, Map>> getOpFieldRemappers() { @@ -129,9 +130,9 @@ public abstract class BaseDriverAdapter extends NBBaseComponent } @Override - public final synchronized StringDriverSpaceCache getSpaceCache() { + public final synchronized ConcurrentSpaceCache getSpaceCache() { if (spaceCache == null) { - spaceCache = new StringDriverSpaceCache<>(this, getSpaceInitializer(getConfiguration())); + spaceCache = new ConcurrentSpaceCache(getSpaceInitializer(getConfiguration())); } return spaceCache; } @@ -192,8 +193,8 @@ public abstract class BaseDriverAdapter extends NBBaseComponent @Override public LongFunction getSpaceFunc(ParsedOp pop) { - LongFunction spaceNameF = pop.getAsFunctionOr("space", "default"); - StringDriverSpaceCache cache = getSpaceCache(); - return l -> getSpaceCache().get(spaceNameF.apply(l)); + LongToIntFunction spaceIdxF = pop.getAsFunctionOrInt("space", 0); + ConcurrentSpaceCache cache = getSpaceCache(); + return l -> getSpaceCache().get(spaceIdxF.applyAsInt(l)); } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java new file mode 100644 index 000000000..3b6aa280c --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java @@ -0,0 +1,36 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import java.util.function.IntFunction; +import java.util.function.LongFunction; + +public class BaseSpace implements Space { + + private final String spaceName; + + public BaseSpace(long idx) { + this.spaceName = String.valueOf(idx); + } + + @Override + public String getName() { + return spaceName; + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java index a05adeb14..d55603f57 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; +import java.util.function.IntFunction; import java.util.function.LongFunction; /** @@ -55,7 +56,7 @@ import java.util.function.LongFunction; * during construction of R type operations, or even for individual * operations. */ -public interface DriverAdapter extends NBComponent { +public interface DriverAdapter extends NBComponent { /** *

@@ -101,7 +102,7 @@ public interface DriverAdapter extends NBComponent * * @return a synthesizer function for {@link OPTYPE} op generation */ - OpMapper getOpMapper(); + OpMapper getOpMapper(); /** * The preprocessor function allows the driver adapter to remap @@ -159,7 +160,7 @@ public interface DriverAdapter extends NBComponent * @return A function which can initialize a new Space, which is a place to hold * object state related to retained objects for the lifetime of a native driver. */ - default Function getSpaceInitializer(NBConfiguration cfg) { + default LongFunction getSpaceInitializer(NBConfiguration cfg) { return n -> null; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java new file mode 100644 index 000000000..6ee1961cb --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java @@ -0,0 +1,37 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.nb.api.components.core.NBNamedElement; + +/** + *

A space is simply a separate namespace associated with an instance of a + * native client or driver. This allows for the emulation of many clients + * in testing scenarios. Within the operations for an adapter, the space + * may be needed, for example, to construct prepared statements, or other + * 'session-attached' objects. Put any state that you would normally + * associate with an instance of a native driver into a space, and use + * the {@link DriverAdapter#getSpaceCache()} to access it when needed.

+ */ +public interface Space extends NBNamedElement, AutoCloseable { + + @Override + default void close() throws Exception { + } +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 4765f161f..c841f47cb 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -23,6 +23,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; @@ -58,6 +59,7 @@ import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.function.Function; +import java.util.function.LongFunction; /** * A default implementation of an Activity, suitable for building upon. @@ -395,7 +397,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok protected OpSequence> createOpSourceFromParsedOps( // Map> adapterCache, // Map> mapperCache, - List> adapters, + List> adapters, List pops ) { try { @@ -424,10 +426,10 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok continue; } - DriverAdapter adapter = adapters.get(i); - OpMapper opMapper = adapter.getOpMapper(); - OpDispenser dispenser = opMapper.apply(pop); - + DriverAdapter adapter = adapters.get(i); + OpMapper opMapper = adapter.getOpMapper(); + LongFunction spaceFunc = adapter.getSpaceFunc(pop); + OpDispenser dispenser = opMapper.apply(pop, spaceFunc); String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none"); if ("op".equalsIgnoreCase(dryrunSpec)) { dispenser = new DryRunOpDispenserWrapper((DriverAdapter) adapter, pop, dispenser); diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index d3154d15d..b2a297021 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -23,6 +23,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -62,7 +63,7 @@ import java.util.concurrent.ConcurrentHashMap; public class StandardActivity extends SimpleActivity implements SyntheticOpTemplateProvider, ActivityDefObserver { private static final Logger logger = LogManager.getLogger("ACTIVITY"); private final OpSequence> sequence; - private final ConcurrentHashMap> adapters = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> adapters = new ConcurrentHashMap<>(); public StandardActivity(NBComponent parent, ActivityDef activityDef) { super(parent, activityDef); @@ -92,11 +93,11 @@ public class StandardActivity extends SimpleActivity implements List pops = new ArrayList<>(); - List> adapterlist = new ArrayList<>(); + List> adapterlist = new ArrayList<>(); NBConfigModel supersetConfig = ConfigModel.of(StandardActivity.class).add(yamlmodel); Optional defaultDriverOption = defaultDriverName; - ConcurrentHashMap> mappers = new ConcurrentHashMap<>(); + ConcurrentHashMap> mappers = new ConcurrentHashMap<>(); for (OpTemplate ot : opTemplates) { // ParsedOp incompleteOpDef = new ParsedOp(ot, NBConfiguration.empty(), List.of(), this); String driverName = ot.getOptionalStringParam("driver", String.class) @@ -112,7 +113,7 @@ public class StandardActivity extends SimpleActivity implements // HERE if (!adapters.containsKey(driverName)) { - DriverAdapter adapter = Optional.of(driverName) + DriverAdapter adapter = Optional.of(driverName) .flatMap( name -> ServiceSelector.of( name, @@ -144,7 +145,7 @@ public class StandardActivity extends SimpleActivity implements supersetConfig.assertValidConfig(activityDef.getParams().getStringStringMap()); - DriverAdapter adapter = adapters.get(driverName); + DriverAdapter adapter = adapters.get(driverName); adapterlist.add(adapter); ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this); Optional discard = pop.takeOptionalStaticValue("driver", String.class); @@ -261,19 +262,20 @@ public class StandardActivity extends SimpleActivity implements */ @Override public void shutdownActivity() { - for (Map.Entry> entry : adapters.entrySet()) { + for (Map.Entry> entry : adapters.entrySet()) { String adapterName = entry.getKey(); DriverAdapter adapter = entry.getValue(); - adapter.getSpaceCache().getElements().forEach((spaceName, space) -> { - if (space instanceof AutoCloseable autocloseable) { + for (Space space : adapter.getSpaceCache()) { + if (space instanceof AutoCloseable autoCloseable) { try { - autocloseable.close(); + // TODO This should be invariant now, remove conditional? + autoCloseable.close(); } catch (Exception e) { throw new RuntimeException("Error while shutting down state space for " + - "adapter=" + adapterName + ", space=" + spaceName + ": " + e, e); + "adapter=" + adapterName + ", space=" + space.getName() + ": " + e, e); } } - }); + } } } From d5ec597152bc52fa57db236676ed686eee51c235 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Oct 2024 13:40:00 -0500 Subject: [PATCH 07/27] add primitive int lambda for op fields --- .../adapters/api/templating/ParsedOp.java | 4 + .../api/templating/ParsedTemplateMap.java | 79 ++++++++++++------- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java index 1d7f43595..3c27192b3 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java @@ -43,6 +43,7 @@ import java.util.*; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongFunction; +import java.util.function.LongToIntFunction; /** *

ParsedOp API

@@ -662,6 +663,9 @@ public class ParsedOp extends NBBaseComponent implements LongFunction LongFunction getAsFunctionOr(String name, V defaultValue) { return tmap.getAsFunctionOr(name, defaultValue); } + public LongToIntFunction getAsFunctionOrInt(String name, int defaultValue) { + return tmap.getAsFunctionOrInt(name, defaultValue); + } /** * Get a LongFunction that first creates a LongFunction of String as in diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java index 91958258a..68c097240 100644 --- a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java @@ -38,6 +38,7 @@ import org.apache.logging.log4j.Logger; import java.util.*; import java.util.function.Function; import java.util.function.LongFunction; +import java.util.function.LongToIntFunction; /** * A parsed map template, which allows construction of extracted or projected functions related @@ -151,7 +152,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } }); Map submap = (Map) v; - ParsedTemplateMap subtpl = new ParsedTemplateMap(getName(),submap, bindings, cfgsources); + ParsedTemplateMap subtpl = new ParsedTemplateMap(getName(), submap, bindings, cfgsources); this.captures.addAll(subtpl.getCaptures()); if (subtpl.isStatic()) { statics.put(k, submap); @@ -168,8 +169,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi statics.put(k, sublist); protomap.put(k, sublist); } else { - dynamics.put(k,subtpl); - protomap.put(k,null); + dynamics.put(k, subtpl); + protomap.put(k, null); } } else { // Eventually, nested and mixed static dynamic structure could be supported, but @@ -215,11 +216,11 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } public Map getConfigPrototype() { - Map cfgs = new LinkedHashMap<>(); + Map cfgs = new LinkedHashMap<>(); for (Map cfgsource : cfgsources) { for (String key : cfgsource.keySet()) { if (!cfgs.containsKey(key)) { - cfgs.put(key,cfgsource.get(key)); + cfgs.put(key, cfgsource.get(key)); } else { logger.warn("config sources contain overlapping keys for '" + key + "', precedence is undefined"); } @@ -244,7 +245,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi public Map applyFull(long value) { Map newmap = apply(value); - for (int i = cfgsources.size()-1; i>0 ; i--) { + for (int i = cfgsources.size() - 1; i > 0; i--) { newmap.putAll(cfgsources.get(i)); } return newmap; @@ -279,7 +280,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi @Override public boolean isDefined(String... fields) { for (String field : fields) { - if (!isStatic(field)&&!isDynamic(field)&&!isConfig(field)) { + if (!isStatic(field) && !isDynamic(field) && !isConfig(field)) { return false; } } @@ -312,7 +313,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi return (T) cfgsource.get(field); } } - throw new OpConfigError("config value for '" +field +"' was not found in " + cfgsources); + throw new OpConfigError("config value for '" + field + "' was not found in " + cfgsources); } public T takeStaticValue(String field, Class classOfT) { @@ -395,11 +396,11 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi public T getStaticConfig(String name, Class clazz) { if (statics.containsKey(name)) { - return NBTypeConverter.convert(statics.get(name),clazz); + return NBTypeConverter.convert(statics.get(name), clazz); } for (Map cfgsource : cfgsources) { if (cfgsource.containsKey(name)) { - return NBTypeConverter.convert(cfgsource.get(name),clazz); + return NBTypeConverter.convert(cfgsource.get(name), clazz); } } if (dynamics.containsKey(name)) { @@ -575,7 +576,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi Object cfgval = getConfig(name); if (type.isAssignableFrom(cfgval.getClass())) { return Optional.of(l -> type.cast(cfgval)); - } else if (NBTypeConverter.canConvert(cfgval,type)) { + } else if (NBTypeConverter.canConvert(cfgval, type)) { return Optional.of(l -> NBTypeConverter.convert(cfgval, type)); } else { throw new OpConfigError( @@ -618,6 +619,25 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } } + public LongToIntFunction getAsFunctionOrInt(String name, int defaultValue) { + if (isDynamic(name)) { + LongFunction f = dynamics.get(name); + Object testValue = f.apply(0); + if (!testValue.getClass().isPrimitive()) { + throw new OpConfigError(STR."getAsFunctionOrInt returned non primitive type: \{testValue.getClass().getCanonicalName()}"); + } + if (!testValue.getClass().equals(int.class)) { + throw new OpConfigError(STR."getAsFunctionOrInt returned non-int type: \{testValue.getClass().getCanonicalName()}"); + } + return (long i) -> (int) f.apply(i); + } else if (isStatic(name) || isConfig(name)) { + int v = (int) getStaticValue(name); + return l -> v; + } else { + return l -> defaultValue; + } + } + /** * Get a LongFunction that first creates a LongFunction of String as in {@link #getAsFunctionOr(String, Object)} )}, but then * applies the result and cached it for subsequent access. This relies on {@link ObjectCache} internally. @@ -765,15 +785,15 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi * @param fieldname the field to take the templates from * @return A map of templates, or an empty map if the field is not defined or is empty. */ - public Map takeAsNamedTemplates(String fieldname) { + public Map takeAsNamedTemplates(String fieldname) { Object entry = originalTemplateObject.get(fieldname); - if (entry !=null) { + if (entry != null) { dynamics.remove(fieldname); statics.remove(fieldname); protomap.remove(fieldname); } - if (entry==null) { + if (entry == null) { for (Map cfgsource : cfgsources) { if (cfgsource.containsKey(fieldname)) { entry = cfgsource.get(fieldname); @@ -782,27 +802,27 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } } - if (entry==null) { + if (entry == null) { return Map.of(); } - Map elements = new LinkedHashMap<>(); + Map elements = new LinkedHashMap<>(); if (entry instanceof CharSequence chars) { - elements.put(this.getName()+"-verifier-0",chars.toString()); + elements.put(this.getName() + "-verifier-0", chars.toString()); } else if (entry instanceof List list) { for (int i = 0; i < list.size(); i++) { - elements.put(this.getName()+"-verifier-"+i,list.get(0)); + elements.put(this.getName() + "-verifier-" + i, list.get(0)); } } else if (entry instanceof Map map) { - map.forEach((k,v) -> { - elements.put(this.getName()+"-verifier-"+k,v); + map.forEach((k, v) -> { + elements.put(this.getName() + "-verifier-" + k, v); }); } - Map parsedStringTemplates + Map parsedStringTemplates = new LinkedHashMap<>(); - elements.forEach((k,v) -> { + elements.forEach((k, v) -> { if (v instanceof CharSequence chars) { - parsedStringTemplates.put(k,new ParsedTemplateString(chars.toString(), this.bindings)); + parsedStringTemplates.put(k, new ParsedTemplateString(chars.toString(), this.bindings)); } }); return parsedStringTemplates; @@ -903,7 +923,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi public Class getValueType(String fieldname) { if (isDynamic(fieldname)) { - return get(fieldname,1).getClass(); + return get(fieldname, 1).getClass(); } if (isStatic(fieldname)) { return getStaticValue(fieldname).getClass(); @@ -1095,12 +1115,12 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi public Map parseStaticCmdMap(String taskname, String mainField) { Object mapsrc = getStaticValue(taskname); - return new LinkedHashMap(ParamsParser.parseToMap(mapsrc,mainField)); + return new LinkedHashMap(ParamsParser.parseToMap(mapsrc, mainField)); } public List> parseStaticCmdMaps(String key, String mainField) { Object mapsSrc = getStaticValue(key); - List> maps = new ArrayList<>(); + List> maps = new ArrayList<>(); for (String spec : mapsSrc.toString().split("; +")) { LinkedHashMap map = new LinkedHashMap<>(ParamsParser.parseToMap(spec, mainField)); maps.add(map); @@ -1118,7 +1138,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi .append(k) .append("->") .append( - v ==null? originalTemplateObject.get(k) : v.toString() + v == null ? originalTemplateObject.get(k) : v.toString() ).append("\n"); } @@ -1135,11 +1155,12 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } - public Map getCombinedPrototype() { - Map prototype = new LinkedHashMap<>(); + public Map getCombinedPrototype() { + Map prototype = new LinkedHashMap<>(); prototype.putAll(getDynamicPrototype()); prototype.putAll(getStaticPrototype()); prototype.putAll(getConfigPrototype()); return prototype; } + } From 05c6b82bbadb8b56d9357d1dd674a3ebed722ca4 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Oct 2024 13:40:09 -0500 Subject: [PATCH 08/27] refactor dryrun for type clarity --- .../uniform/opwrappers/DryCycleOp.java | 34 +++++++++ .../DryCycleOpDispenserWrapper.java | 45 ++++++++++++ .../{DryRunOp.java => DryRunableOp.java} | 8 +- ...ava => DryRunnableOpDispenserWrapper.java} | 20 +++-- .../uniform/opwrappers/EmitterCycleOp.java | 34 +++++++++ .../EmitterCycleOpDispenserWrapper.java | 45 ++++++++++++ .../uniform/opwrappers/EmitterOp.java | 10 +-- .../opwrappers/EmitterOpDispenserWrapper.java | 5 +- .../uniform/opwrappers/EmitterRunnableOp.java | 32 ++++++++ .../EmitterRunnableOpDispenserWrapper.java | 45 ++++++++++++ .../engine/api/activityimpl/OpWrappers.java | 73 +++++++++++++++++++ .../api/activityimpl/SimpleActivity.java | 22 ++---- 12 files changed, 339 insertions(+), 34 deletions(-) create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{DryRunOp.java => DryRunableOp.java} (83%) rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{DryRunOpDispenserWrapper.java => DryRunnableOpDispenserWrapper.java} (67%) create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOp.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOpDispenserWrapper.java create mode 100644 nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java new file mode 100644 index 000000000..549ac043e --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; + +public class DryCycleOp implements CycleOp { + + private final CycleOp op; + + public DryCycleOp(CycleOp op) { + this.op = op; + } + + @Override + public T apply(long value) { + return null; + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java new file mode 100644 index 000000000..894126f7e --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +public class DryCycleOpDispenserWrapper extends BaseOpDispenser, S> { + + private final OpDispenser> realDispenser; + + public DryCycleOpDispenserWrapper( + DriverAdapter, S> adapter, + ParsedOp pop, + OpDispenser> realDispenser + ) { + super(adapter, pop); + this.realDispenser = realDispenser; + } + + @Override + public CycleOp getOp(long cycle) { + CycleOp op = realDispenser.getOp(cycle); + return new DryCycleOp<>(op); + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunableOp.java similarity index 83% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunOp.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunableOp.java index a5775f10a..2d541bb7f 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunableOp.java @@ -16,16 +16,16 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; -import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; -public class DryRunOp implements RunnableOp { +public class DryRunableOp implements RunnableOp { - private final Op op; + private final RunnableOp op; - public DryRunOp(Op op) { + public DryRunableOp(RunnableOp op) { this.op = op; } + @Override public void run() { } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunnableOpDispenserWrapper.java similarity index 67% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunOpDispenserWrapper.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunnableOpDispenserWrapper.java index 55a514d38..d4388ff08 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunOpDispenserWrapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryRunnableOpDispenserWrapper.java @@ -19,20 +19,26 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; import io.nosqlbench.adapters.api.templating.ParsedOp; -public class DryRunOpDispenserWrapper extends BaseOpDispenser { +public class DryRunnableOpDispenserWrapper extends BaseOpDispenser { - private final OpDispenser realDispenser; + private final OpDispenser realDispenser; - public DryRunOpDispenserWrapper(DriverAdapter adapter, ParsedOp pop, OpDispenser realDispenser) { + public DryRunnableOpDispenserWrapper( + DriverAdapter adapter, + ParsedOp pop, + OpDispenser realDispenser + ) { super(adapter, pop); this.realDispenser = realDispenser; } + @Override - public DryRunOp getOp(long cycle) { - Op op = realDispenser.getOp(cycle); - return new DryRunOp(op); + public DryRunableOp getOp(long cycle) { + RunnableOp op = realDispenser.getOp(cycle); + return new DryRunableOp(op); } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java new file mode 100644 index 000000000..16511cedb --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; + +public class EmitterCycleOp implements CycleOp { + + private final CycleOp cycleOp; + public EmitterCycleOp(CycleOp cycleOp) { + this.cycleOp = cycleOp; + } + + @Override + public T apply(long value) { + T result = cycleOp.apply(value); + System.out.println("result from cycle " + value + ":\n"+result); + return result; + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java new file mode 100644 index 000000000..390909965 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +public class EmitterCycleOpDispenserWrapper extends BaseOpDispenser, S> { + + private final OpDispenser> realDispenser; + + public EmitterCycleOpDispenserWrapper( + DriverAdapter, S> adapter, + ParsedOp pop, + OpDispenser> realDispenser + ) { + super(adapter, pop); + this.realDispenser = realDispenser; + } + + @Override + public EmitterCycleOp getOp(long cycle) { + CycleOp cycleOp = realDispenser.getOp(cycle); + return new EmitterCycleOp(cycleOp); + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java index 8e03fa81c..d009bfdf1 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java @@ -18,16 +18,16 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; -public class EmitterOp implements CycleOp { +public class EmitterOp implements CycleOp { - private final CycleOp cycleOp; - public EmitterOp(CycleOp cycleOp) { + private final CycleOp cycleOp; + public EmitterOp(CycleOp cycleOp) { this.cycleOp = cycleOp; } @Override - public Object apply(long value) { - Object result = cycleOp.apply(value); + public T apply(long value) { + T result = cycleOp.apply(value); System.out.println("result from cycle " + value + ":\n"+result); return result; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java index 292e77697..904e964a2 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java @@ -19,15 +19,16 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.templating.ParsedOp; -public class EmitterOpDispenserWrapper extends BaseOpDispenser { +public class EmitterOpDispenserWrapper extends BaseOpDispenser { private final OpDispenser> realDispenser; - public EmitterOpDispenserWrapper(DriverAdapter adapter, ParsedOp pop, OpDispenser> realDispenser) { + public EmitterOpDispenserWrapper(DriverAdapter adapter, ParsedOp pop, OpDispenser> realDispenser) { super(adapter, pop); this.realDispenser = realDispenser; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOp.java new file mode 100644 index 000000000..b0ba625c3 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOp.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; + +public class EmitterRunnableOp implements RunnableOp { + + private final RunnableOp runnableOp; + public EmitterRunnableOp(RunnableOp runnableOp) { + this.runnableOp = runnableOp; + } + + @Override + public void run() { + runnableOp.run(); + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOpDispenserWrapper.java new file mode 100644 index 000000000..647a9cc62 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterRunnableOpDispenserWrapper.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +public class EmitterRunnableOpDispenserWrapper extends BaseOpDispenser { + + private final OpDispenser realDispenser; + + public EmitterRunnableOpDispenserWrapper( + DriverAdapter adapter, + ParsedOp pop, + OpDispenser realDispenser + ) { + super(adapter, pop); + this.realDispenser = realDispenser; + } + + @Override + public EmitterRunnableOp getOp(long cycle) { + RunnableOp cycleOp = realDispenser.getOp(cycle); + return new EmitterRunnableOp(cycleOp); + } +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java new file mode 100644 index 000000000..205c81ae6 --- /dev/null +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java @@ -0,0 +1,73 @@ +package io.nosqlbench.engine.api.activityimpl; + +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.DryCycleOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.DryRunnableOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.EmitterCycleOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.EmitterRunnableOpDispenserWrapper; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.nb.api.errors.OpConfigError; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class OpWrappers { + + public final static Logger logger = LogManager.getLogger(OpWrappers.class); + + public static OpDispenser wrapOptionally( + DriverAdapter adapter, + OpDispenser dispenser, + ParsedOp pop, + String dryrunSpec + ) { + if (dryrunSpec.isEmpty() || "none".equals(dryrunSpec)) { + return dispenser; + } + + + if ("op".equalsIgnoreCase(dryrunSpec)) { + Op exampleOp = dispenser.getOp(0L); + + if (exampleOp instanceof RunnableOp runnableOp) { + dispenser = new DryRunnableOpDispenserWrapper(adapter, pop, dispenser); + } else if (exampleOp instanceof CycleOp cycleOp) { + dispenser = new DryCycleOpDispenserWrapper(adapter, pop, dispenser); + } else { + throw new OpConfigError("Unable to wrap op named '" + pop.getDefinedNames() + "' for dry run, since" + + "only RunnableOp and CycleOp types are supported"); + } + logger.warn( + "initialized {} for dry run only. " + + "This op will be synthesized for each cycle, but will not be executed.", + pop.getName() + ); + + } else if ("emit".equalsIgnoreCase(dryrunSpec)) { + Op exampleOp = dispenser.getOp(0L); + if (exampleOp instanceof RunnableOp runnableOp) { + dispenser = new EmitterRunnableOpDispenserWrapper(adapter, pop, dispenser); + } else if (exampleOp instanceof CycleOp cycleOp) { + dispenser = new EmitterCycleOpDispenserWrapper(adapter, pop, dispenser); + } else { + throw new OpConfigError("Unable to make op named '" + pop.getName() + "' emit a value, " + + "since only RunnableOp and CycleOp types are supported"); + } + dispenser = new EmitterRunnableOpDispenserWrapper( + (DriverAdapter) adapter, + pop, + (OpDispenser) dispenser + ); + logger.warn( + "initialized {} for to emit the result type to stdout. ", + pop.getName() + ); + + } + return dispenser; + } +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index c841f47cb..06736c3ab 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -27,8 +27,10 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; -import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.DryRunOpDispenserWrapper; -import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.EmitterOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.DryRunnableOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.EmitterCycleOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.EmitterRunnableOpDispenserWrapper; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.activityapi.core.*; import io.nosqlbench.engine.api.activityapi.core.progress.ActivityMetricProgressMeter; @@ -415,7 +417,6 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok .orElse(SequencerType.bucket); SequencePlanner> planner = new SequencePlanner<>(sequencerType); - int dryrunCount = 0; for (int i = 0; i < pops.size(); i++) { long ratio = ratios.get(i); ParsedOp pop = pops.get(i); @@ -431,16 +432,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok LongFunction spaceFunc = adapter.getSpaceFunc(pop); OpDispenser dispenser = opMapper.apply(pop, spaceFunc); String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none"); - if ("op".equalsIgnoreCase(dryrunSpec)) { - dispenser = new DryRunOpDispenserWrapper((DriverAdapter) adapter, pop, dispenser); - dryrunCount++; - } else if ("emit".equalsIgnoreCase(dryrunSpec)) { - dispenser = new EmitterOpDispenserWrapper( - (DriverAdapter) adapter, - pop, - (OpDispenser>) dispenser - ); - } + dispenser = OpWrappers.wrapOptionally(adapter, dispenser, pop, dryrunSpec); // if (strict) { // optemplate.assertConsumed(); @@ -450,9 +442,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok throw new OpConfigError("Error while mapping op from template named '" + pop.getName() + "': " + e.getMessage(), e); } } - if (0 < dryrunCount) { - logger.warn("initialized {} op templates for dry run only. These ops will be synthesized for each cycle, but will not be executed.", dryrunCount); - } + return planner.resolve(); From 09e3881fd5835981927408017039e8369cc45977 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Oct 2024 13:40:18 -0500 Subject: [PATCH 09/27] modularize new adapters --- mvn-defaults/pom.xml | 23 ----------------------- nb-adapters/adapter-weaviate/pom.xml | 3 +++ nb-adapters/nb-adapters-included/pom.xml | 2 +- nb-adapters/pom.xml | 20 ++++++++++---------- 4 files changed, 14 insertions(+), 34 deletions(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 7a9d3eb30..76d679cf8 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -398,29 +398,6 @@ 2.24.1 - - io.weaviate - client - 4.8.2 - - - - com.azure - azure-search-documents - 11.7.0 - - - com.azure - azure-identity - 1.13.2 - - - - com.google.cloud - google-cloud-spanner - 6.77.0 - - diff --git a/nb-adapters/adapter-weaviate/pom.xml b/nb-adapters/adapter-weaviate/pom.xml index 6df5514d6..13c99ddb5 100644 --- a/nb-adapters/adapter-weaviate/pom.xml +++ b/nb-adapters/adapter-weaviate/pom.xml @@ -32,9 +32,12 @@ ${revision} compile + io.weaviate client + 4.8.2 + diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 7c854e88e..1173ae93a 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -270,7 +270,7 @@ adapter-weaviate-include - true + false diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index fc27812a8..e51c61762 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -194,16 +194,6 @@ - - adapter-weaviate-module - - true - - - adapter-weaviate - - - adapter-azure-aisearch-module @@ -224,5 +214,15 @@ + + adapter-weaviate-module + + false + + + adapter-weaviate + + + From f68895449c77fcd90bcd4b1740fb46d060dac284 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 23 Oct 2024 13:40:37 -0500 Subject: [PATCH 10/27] version bumps --- nb-docsys/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/nb-docsys/pom.xml b/nb-docsys/pom.xml index feece3dde..40ad86f99 100644 --- a/nb-docsys/pom.xml +++ b/nb-docsys/pom.xml @@ -42,6 +42,7 @@ + org.eclipse.jetty jetty-server From 3bcda60485354b4d0d58bab317f361bcd2756646 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 24 Oct 2024 14:19:46 -0500 Subject: [PATCH 11/27] align adapter-cqld4 to API changes --- .../adapter/cqld4/Cqld4DriverAdapter.java | 12 ++-- .../nosqlbench/adapter/cqld4/Cqld4Space.java | 10 +-- .../opdispensers/CqlD4BatchStmtDispenser.java | 19 ++--- .../CqlD4RainbowTableDispenser.java | 8 ++- .../opdispensers/Cqld4BaseOpDispenser.java | 10 ++- .../Cqld4FluentGraphOpDispenser.java | 24 ++++--- .../opdispensers/Cqld4GremlinOpDispenser.java | 8 ++- .../Cqld4PreparedStmtDispenser.java | 9 ++- .../opdispensers/Cqld4RawStmtDispenser.java | 10 ++- .../Cqld4SimpleCqlStmtDispenser.java | 9 ++- .../cqld4/opmappers/CqlD4BatchStmtMapper.java | 34 ++++----- .../opmappers/CqlD4CqlSimpleStmtMapper.java | 18 +++-- .../opmappers/CqlD4PreparedStmtMapper.java | 23 +++---- .../opmappers/CqlD4RainbowTableMapper.java | 17 +++-- .../cqld4/opmappers/CqlD4RawStmtMapper.java | 28 +++++--- .../cqld4/opmappers/Cqld4BaseOpMapper.java | 52 ++++++++++++++ .../cqld4/opmappers/Cqld4CoreOpMapper.java | 53 ++++++-------- .../cqld4/opmappers/Cqld4CqlOpMapper.java | 69 +++++++++++++++++++ .../opmappers/Cqld4FluentGraphOpMapper.java | 30 +++++--- .../cqld4/opmappers/Cqld4GremlinOpMapper.java | 17 +++-- .../adapter/cqld4/optypes/Cqld4BaseOp.java | 6 ++ .../adapter/cqld4/optypes/Cqld4CqlOp.java | 4 +- .../cqld4/optypes/Cqld4FluentGraphOp.java | 2 +- .../cqld4/optypes/Cqld4RainbowTableOp.java | 9 +-- .../cqld4/optypes/Cqld4ScriptGraphOp.java | 2 +- 25 files changed, 331 insertions(+), 152 deletions(-) create mode 100644 nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4BaseOpMapper.java create mode 100644 nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlOpMapper.java create mode 100644 nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4BaseOp.java diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java index b279067a2..10822f1a6 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java @@ -17,6 +17,8 @@ package io.nosqlbench.adapter.cqld4; import io.nosqlbench.adapter.cqld4.opmappers.Cqld4CoreOpMapper; +import io.nosqlbench.adapter.cqld4.optypes.Cqld4BaseOp; +import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; import io.nosqlbench.nb.api.config.standard.NBConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpMapper; @@ -35,9 +37,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.LongFunction; @Service(value = DriverAdapter.class, selector = "cqld4") -public class Cqld4DriverAdapter extends BaseDriverAdapter { +public class Cqld4DriverAdapter extends BaseDriverAdapter { private final static Logger logger = LogManager.getLogger(Cqld4DriverAdapter.class); public Cqld4DriverAdapter(NBComponent parentComponent, NBLabels labels) { @@ -45,14 +48,13 @@ public class Cqld4DriverAdapter extends BaseDriverAdapter { } @Override - public OpMapper getOpMapper() { - StringDriverSpaceCache spaceCache = getSpaceCache(); + public OpMapper getOpMapper() { NBConfiguration config = getConfiguration(); - return new Cqld4CoreOpMapper(this, config, spaceCache); + return new Cqld4CoreOpMapper(this, config); } @Override - public Function getSpaceInitializer(NBConfiguration cfg) { + public LongFunction getSpaceInitializer(NBConfiguration cfg) { return s -> new Cqld4Space(s,cfg); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java index f6bf017c9..9ec1f1cad 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java @@ -28,6 +28,7 @@ import io.nosqlbench.adapter.cqld4.optionhelpers.OptionHelpers; import io.nosqlbench.adapter.cqld4.wrapper.Cqld4LoadBalancerObserver; import io.nosqlbench.adapter.cqld4.wrapper.Cqld4SessionBuilder; import io.nosqlbench.adapter.cqld4.wrapper.NodeSummary; +import io.nosqlbench.adapters.api.activityimpl.uniform.BaseSpace; import io.nosqlbench.nb.api.config.standard.*; import io.nosqlbench.nb.api.errors.OpConfigError; import io.nosqlbench.nb.api.nbio.Content; @@ -47,14 +48,13 @@ import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; -public class Cqld4Space implements AutoCloseable { +public class Cqld4Space extends BaseSpace { private final static Logger logger = LogManager.getLogger(Cqld4Space.class); - private final String space; CqlSession session; - public Cqld4Space(String space, NBConfiguration cfg) { - this.space = space; + public Cqld4Space(long space, NBConfiguration cfg) { + super(space); session = createSession(cfg); } @@ -340,7 +340,7 @@ public class Cqld4Space implements AutoCloseable { try { this.getSession().close(); } catch (Exception e) { - logger.warn("auto-closeable cql session threw exception in cql space(" + this.space + "): " + e); + logger.warn("auto-closeable cql session threw exception in cql space(" + getName() + "): " + e); throw e; } } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4BatchStmtDispenser.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4BatchStmtDispenser.java index 74501e223..fe77fff31 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4BatchStmtDispenser.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4BatchStmtDispenser.java @@ -18,25 +18,28 @@ package io.nosqlbench.adapter.cqld4.opdispensers; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.cql.*; +import io.nosqlbench.adapter.cqld4.Cqld4DriverAdapter; +import io.nosqlbench.adapter.cqld4.Cqld4Space; import io.nosqlbench.adapter.cqld4.optionhelpers.BatchTypeEnum; import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlBatchStatement; import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.BaseSpace; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.jetbrains.annotations.NotNull; import java.util.function.LongFunction; -public class CqlD4BatchStmtDispenser extends Cqld4BaseOpDispenser { +public class CqlD4BatchStmtDispenser extends Cqld4BaseOpDispenser { private final int repeat; private final ParsedOp subop; private final OpMapper submapper; private LongFunction opfunc; public CqlD4BatchStmtDispenser( - DriverAdapter adapter, + Cqld4DriverAdapter adapter, LongFunction sessionFunc, ParsedOp op, int repeat, @@ -48,7 +51,7 @@ public class CqlD4BatchStmtDispenser extends Cqld4BaseOpDispenser { this.subop = subop; this.opfunc = createStmtFunc(op, subopDispenser); this.submapper = adapter.getOpMapper(); - subopDispenser = submapper.apply(subop); + subopDispenser = submapper.apply(subop, adapter.getSpaceFunc(op)); } @@ -62,19 +65,19 @@ public class CqlD4BatchStmtDispenser extends Cqld4BaseOpDispenser { BatchTypeEnum bte = topOp.getEnumFromFieldOr(BatchTypeEnum.class, BatchTypeEnum.unlogged, "batchtype"); LongFunction bsbf = l -> new BatchStatementBuilder(bte.batchtype); LongFunction bsf = getBatchAccumulator(bsbf, subopDispenser); - bsf = getEnhancedStmtFunc(bsf,topOp); + bsf = getEnhancedStmtFunc(bsf, topOp); return bsf; } @NotNull private LongFunction getBatchAccumulator(LongFunction bsb, OpDispenser subopDispenser) { LongFunction f = l -> { - long base=l*repeat; + long base = l * repeat; BatchStatementBuilder bsa = bsb.apply(l); for (int i = 0; i < repeat; i++) { - Cqld4CqlOp op = subopDispenser.apply(base+i); + Cqld4CqlOp op = subopDispenser.apply(base + i); BatchableStatement stmt = (BatchableStatement) op.getStmt(); - bsa= bsa.addStatement(stmt); + bsa = bsa.addStatement(stmt); } return bsa; }; @@ -84,7 +87,7 @@ public class CqlD4BatchStmtDispenser extends Cqld4BaseOpDispenser { } @Override - public Cqld4CqlOp getOp(long value) { + public Cqld4CqlBatchStatement getOp(long value) { Statement bstmt = opfunc.apply(value); return new Cqld4CqlBatchStatement( getSessionFunc().apply(value), diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4RainbowTableDispenser.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4RainbowTableDispenser.java index 41e05e134..38ae865ca 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4RainbowTableDispenser.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/CqlD4RainbowTableDispenser.java @@ -17,18 +17,22 @@ package io.nosqlbench.adapter.cqld4.opdispensers; import com.datastax.oss.driver.api.core.CqlSession; +import io.nosqlbench.adapter.cqld4.Cqld4DriverAdapter; +import io.nosqlbench.adapter.cqld4.Cqld4Space; import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; +import io.nosqlbench.adapter.cqld4.optypes.Cqld4RainbowTableOp; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class CqlD4RainbowTableDispenser extends Cqld4BaseOpDispenser { +public class CqlD4RainbowTableDispenser extends Cqld4BaseOpDispenser { // private final LongFunction stmtFunc; private final LongFunction targetFunction; - public CqlD4RainbowTableDispenser(DriverAdapter adapter, LongFunction sessionFunc, LongFunction targetFunction, ParsedOp cmd) { + public CqlD4RainbowTableDispenser(Cqld4DriverAdapter adapter, LongFunction sessionFunc, + LongFunction targetFunction, ParsedOp cmd) { super(adapter, sessionFunc,cmd); this.targetFunction=targetFunction; // this.tableFunc =createTableFunc(cmd); diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java index 1b9e17551..5a9e33c57 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java @@ -24,8 +24,10 @@ import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; import com.datastax.oss.driver.api.core.cql.*; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.token.Token; +import io.nosqlbench.adapter.cqld4.Cqld4DriverAdapter; import io.nosqlbench.adapter.cqld4.Cqld4Space; import io.nosqlbench.adapter.cqld4.instruments.CqlOpMetrics; +import io.nosqlbench.adapter.cqld4.optypes.Cqld4BaseOp; import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; @@ -40,19 +42,21 @@ import java.time.Duration; import java.util.Map; import java.util.function.LongFunction; -public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser implements CqlOpMetrics { +public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser implements CqlOpMetrics { private final static Logger logger = LogManager.getLogger("CQLD4"); private final int maxpages; - private final LongFunction sessionFunc; + protected final LongFunction sessionFunc; private final boolean isRetryReplace; private final int maxLwtRetries; private final Histogram rowsHistogram; private final Histogram pagesHistogram; private final Histogram payloadBytesHistogram; - public Cqld4BaseOpDispenser(DriverAdapter adapter, LongFunction sessionFunc, ParsedOp op) { + public Cqld4BaseOpDispenser(Cqld4DriverAdapter adapter, + LongFunction sessionFunc, + ParsedOp op) { super(adapter, op); this.sessionFunc = sessionFunc; this.maxpages = op.getStaticConfigOr("maxpages", 1); diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4FluentGraphOpDispenser.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4FluentGraphOpDispenser.java index 145cd07cb..8c6cdb1af 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4FluentGraphOpDispenser.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4FluentGraphOpDispenser.java @@ -20,11 +20,12 @@ import com.datastax.dse.driver.api.core.graph.FluentGraphStatement; import com.datastax.dse.driver.api.core.graph.FluentGraphStatementBuilder; import com.datastax.oss.driver.api.core.CqlSession; import groovy.lang.Script; +import io.nosqlbench.adapter.cqld4.Cqld4DriverAdapter; import io.nosqlbench.adapter.cqld4.Cqld4Space; +import io.nosqlbench.adapter.cqld4.optypes.Cqld4BaseOp; +import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlOp; import io.nosqlbench.adapter.cqld4.optypes.Cqld4FluentGraphOp; -import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; -import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.virtdata.core.bindings.Bindings; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; @@ -34,7 +35,7 @@ import java.util.Map; import java.util.function.LongFunction; import java.util.function.Supplier; -public class Cqld4FluentGraphOpDispenser extends BaseOpDispenser { +public class Cqld4FluentGraphOpDispenser extends Cqld4BaseOpDispenser { private final LongFunction graphnameFunc; private final LongFunction sessionFunc; @@ -42,22 +43,23 @@ public class Cqld4FluentGraphOpDispenser extends BaseOpDispenser private final ThreadLocal