From ca38a710c3cbe6e60ce81423ace592ee534a3774 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 10 Dec 2024 15:41:05 -0600 Subject: [PATCH] Jshook/verification (#2107) * docs update * typos * disable noisy prom exposition dump * fix refkey bug * move specs into non-test namespace * remove unimplemented rainbow ops * docs updates * give credit to maintainers * update milvus module to build clean on new APIs * remove var keyword * API remapping * enable new op behaviors --- mvn-defaults/pom.xml | 16 + .../nosqlbench/adapter/amqp/AmqpOpMapper.java | 2 +- .../azureaisearch/AzureAISearchOpMapper.java | 4 +- .../azureaisearch/AzureAISearchSpace.java | 24 +- ...ureAISearchUploadDocumentsOpDispenser.java | 2 +- .../adapter/cqld4/Cqld4DriverAdapter.java | 8 +- .../cqld4/opmappers/CqlD4BatchStmtMapper.java | 4 +- .../opmappers/CqlD4CqlSimpleStmtMapper.java | 2 +- .../adapter/cqld4/opmappers/CqlD4OpType.java | 70 +- .../opmappers/CqlD4PreparedStmtMapper.java | 4 +- .../opmappers/CqlD4RainbowTableMapper.java | 2 +- .../cqld4/opmappers/CqlD4RawStmtMapper.java | 2 +- .../cqld4/opmappers/Cqld4BaseOpMapper.java | 2 +- .../cqld4/opmappers/Cqld4CoreOpMapper.java | 4 +- .../cqld4/opmappers/Cqld4CqlBaseOpMapper.java | 2 +- .../cqld4/opmappers/Cqld4CqlOpMapper.java | 10 +- .../opmappers/Cqld4FluentGraphOpMapper.java | 2 +- .../cqld4/opmappers/Cqld4GremlinOpMapper.java | 2 +- .../adapter/cqld4/optypes/Cqld4CqlOp.java | 36 +- .../cqld4/optypes/Cqld4RainbowTableOp.java | 10 +- .../validators/Cqld4SingleRowValidator.java | 35 + .../cqlgen/core/CGWorkloadExporter.java | 2 +- .../src/main/resources/cqlverify.md | 162 ++ .../cql/cql/parser/CqlParserHarnessTest.java | 2 +- .../adapter/dataapi/DataApiOpMapper.java | 2 +- .../adapter/dynamodb/DynamoDBOpMapper.java | 2 +- .../adapter/prototype/ExampleOpMapper.java | 12 +- .../adapter/prototype/ExampleSpace.java | 9 +- .../gcpspanner/GCPSpannerOpMapper.java | 4 +- .../adapter/http/core/HttpOpMapper.java | 4 +- .../adapter/kafka/KafkaOpMapper.java | 2 +- .../adapter/milvus/MilvusDriverAdapter.java | 12 +- .../adapter/milvus/MilvusOpMapper.java | 203 ++- .../adapter/milvus/MilvusSpace.java | 26 +- .../MilvusAlterAliasOpDispenser.java | 15 +- .../MilvusAlterCollectionOpDispenser.java | 7 +- .../opdispensers/MilvusBaseOpDispenser.java | 9 +- .../MilvusBulkInsertOpDispenser.java | 9 +- .../MilvusCreateAliasOpDispenser.java | 9 +- .../MilvusCreateCollectionOpDispenser.java | 21 +- .../MilvusCreateCredentialOpDispenser.java | 7 +- .../MilvusCreateDatabaseOpDispenser.java | 7 +- .../MilvusCreateIndexOpDispenser.java | 21 +- .../MilvusCreatePartitionOpDispenser.java | 9 +- .../MilvusDeleteCredentialOpDispenser.java | 7 +- .../opdispensers/MilvusDeleteOpDispenser.java | 9 +- .../MilvusDescribeCollectionOpDispenser.java | 10 +- .../MilvusDescribeIndexOpDispenser.java | 11 +- .../MilvusDropAliasOpDispenser.java | 7 +- .../MilvusDropCollectionOpDispenser.java | 26 +- .../MilvusDropDatabaseOpDispenser.java | 7 +- .../MilvusDropIndexOpDispenser.java | 29 +- .../MilvusDropPartitionOpDispenser.java | 10 +- .../opdispensers/MilvusFlushOpDispenser.java | 9 +- .../MilvusGetBulkInsertStateOpDispenser.java | 8 +- ...vusGetCollectionStatisticsOpDispenser.java | 9 +- .../MilvusGetCompactionStateOpDispenser.java | 7 +- ...etCompactionStateWithPlansOpDispenser.java | 8 +- .../MilvusGetFlushAllStateOpDispenser.java | 7 +- .../MilvusGetFlushStateOpDispenser.java | 8 +- ...ilvusGetIndexBuildProgressOpDispenser.java | 9 +- .../MilvusGetIndexStateOpDispenser.java | 9 +- .../MilvusGetLoadStateOpDispenser.java | 11 +- .../MilvusGetLoadingProgressOpDispenser.java | 9 +- .../MilvusGetMetricsOpDispenser.java | 7 +- .../opdispensers/MilvusGetOpDispenser.java | 9 +- ...lvusGetPartitionStatisticsOpDispenser.java | 9 +- ...usGetPersistentSegmentInfoOpDispenser.java | 9 +- .../MilvusGetQuerySegmentInfoOpDispenser.java | 7 +- .../MilvusGetReplicasOpDispenser.java | 7 +- .../MilvusHasPartitionOpDispenser.java | 8 +- .../opdispensers/MilvusInsertOpDispenser.java | 29 +- .../MilvusInsertRowsOpDispenser.java | 29 +- .../MilvusListBulkInsertTasksOpDispenser.java | 7 +- .../MilvusListCollectionsOpDispenser.java | 7 +- .../MilvusListCredUsersOpDispenser.java | 7 +- .../MilvusListDatabasesOpDispenser.java | 7 +- .../MilvusLoadBalanceOpDispenser.java | 7 +- .../MilvusLoadCollectionOpDispenser.java | 9 +- .../MilvusLoadPartitionsOpDispenser.java | 18 +- .../MilvusLoadingProgressOpDispenser.java | 9 +- .../MilvusManualCompactOpDispenser.java | 7 +- .../opdispensers/MilvusQueryOpDispenser.java | 9 +- .../MilvusReleaseCollectionOpDispenser.java | 7 +- .../MilvusReleasePartitionsOpDispenser.java | 9 +- .../opdispensers/MilvusSearchOpDispenser.java | 10 +- .../MilvusShowCollectionsOpDispenser.java | 9 +- .../MilvusShowPartitionsOpDispenser.java | 9 +- .../MilvusUpdateCredentialOpDispenser.java | 7 +- .../adapter/milvus/ops/MilvusBaseOp.java | 2 +- .../milvus/ops/MilvusDescribeIndexOp.java | 4 +- .../milvus/ops/MilvusGetLoadStateOp.java | 4 +- .../adapter/mongodb/core/MongoOpMapper.java | 6 +- .../adapter/neo4j/Neo4JOpMapper.java | 2 +- .../adapter/neo4j/ops/Neo4JSyncReadTxnOp.java | 3 +- .../neo4j/ops/Neo4JSyncWriteTxnOp.java | 3 +- .../adapter/pulsar/PulsarOpMapper.java | 2 +- .../adapter/qdrant/QdrantOpMapper.java | 4 +- .../nosqlbench/adapter/s4j/S4JOpMapper.java | 2 +- .../adapter/stdout/StdoutDriverAdapter.java | 2 +- .../adapter/stdout/StdoutOpMapper.java | 2 +- .../adapter/tcpclient/TcpClientOpMapper.java | 2 +- .../adapter/tcpserver/TcpServerOpMapper.java | 2 +- nb-adapters/adapter-weaviate/pom.xml | 2 +- .../weaviate/WeaviateDriverAdapter.java | 34 +- .../adapter/weaviate/WeaviateOpMapper.java | 48 +- .../adapter/weaviate/WeaviateSpace.java | 112 +- .../adapter/weaviate/ops/WeaviateBaseOp.java | 14 +- .../ops/WeaviateCreateCollectionOp.java | 4 +- .../weaviate/ops/WeaviateCreateObjectsOp.java | 4 +- .../ops/WeaviateDeleteCollectionOp.java | 4 +- .../ops/WeaviateGetCollectionSchemaOp.java | 4 +- .../WeaviateBaseOpDispenser.java | 52 +- .../WeaviateCreateCollectionOpDispenser.java | 25 +- .../WeaviateCreateObjectsOpDispenser.java | 13 +- .../WeaviateDeleteCollectionOpDispenser.java | 24 +- ...eaviateGetCollectionSchemaOpDispenser.java | 29 +- .../activities/weaviate_vector_live.yaml | 23 +- nb-adapters/nb-adapters-included/pom.xml | 2 +- nb-adapters/pom.xml | 10 +- nb-apis/adapters-api/pom.xml | 8 + .../api/activityconfig/OpsLoader.java | 40 +- .../api/activityconfig/rawyaml/RawOpDef.java | 43 +- .../activityconfig/rawyaml/RawOpsLoader.java | 11 +- .../api/activityconfig/yaml/OpData.java | 76 +- .../api/activityconfig/yaml/OpDef.java | 20 +- .../api/activityconfig/yaml/OpTemplate.java | 31 +- .../activityconfig/yaml/OpTemplateProps.java | 46 + .../api/activityconfig/yaml/OpsDocList.java | 10 +- .../api/activityimpl/OpDispenser.java | 143 +- .../adapters/api/activityimpl/OpMapper.java | 6 +- .../api/activityimpl/uniform/BaseSpace.java | 3 + .../activityimpl/uniform/DriverAdapter.java | 423 +++-- .../api/activityimpl/uniform/Space.java | 5 +- .../api/activityimpl/uniform/Validator.java | 23 + .../activityimpl/uniform/ValidatorSource.java | 29 + .../uniform/flowtypes/CycleOp.java | 64 +- .../{EmitterCycleOp.java => AssertingOp.java} | 18 +- .../opwrappers/AssertingOpDispenser.java | 49 + .../uniform/opwrappers/CapturingOp.java | 41 + ...Wrapper.java => CapturingOpDispenser.java} | 24 +- .../{DryCycleOp.java => DryrunOp.java} | 4 +- ...serWrapper.java => DryrunOpDispenser.java} | 6 +- .../{EmitterOp.java => ResultPrintingOp.java} | 4 +- ...er.java => ResultPrintingOpDispenser.java} | 8 +- .../api/evalctx/comparators/DiffType.java | 53 + .../evalctx/comparators/MapDifferencer.java | 339 ++++ .../nosqlbench/adapters/api/package-info.java | 20 + .../adapters/api/templating/ParsedOp.java | 1399 +++++++++-------- .../00_workload_specification.md | 0 .../01_spectest_formatting.md | 0 .../02_workload_structure.md | 0 .../04_op_template_basics.md | 0 .../05_op_template_payloads.md | 0 .../06_op_template_variations.md | 0 .../07_template_variables.md | 0 .../workload_definition/08_parsed_op_api.md | 0 .../reporters/PromPushReporterComponent.java | 2 +- .../nosqlbench/nb/api/stats/StatBucket.java | 14 +- .../engine/api/activityimpl/OpAssertions.java | 62 + .../engine/api/activityimpl/OpCapture.java | 67 + .../{OpWrappers.java => OpDryrun.java} | 21 +- .../activityimpl/OpFunctionComposition.java | 77 + .../ResultValidatingDispenser.java | 22 + .../api/activityimpl/SimpleActivity.java | 7 +- .../api/templating/ParsedTemplateMap.java | 664 ++++---- .../virtdata/core/templates/CapturePoint.java | 154 +- .../core/templates/CapturePointException.java | 25 + .../core/templates/CapturePointParser.java | 71 +- .../core/templates/CapturePoints.java | 37 + .../templates/DynamicVariableCapture.java | 18 +- .../core/templates/ParsedTemplateString.java | 7 +- .../templates/UniformVariableCapture.java | 43 + .../templates/CapturePointParserTest.java | 45 +- .../core/templates/CapturePointTest.java | 6 +- .../examples/injava/NB_activity_error.java | 10 +- .../examples/injava/NB_await_finished.java | 11 +- .../nbr/examples/injava/NB_basicdiag.java | 5 +- .../injava/NB_cocycledelay_bursty_backup.java | 11 +- .../injava/NB_extension_csvoutput.java | 6 - .../nbr/examples/injava/NB_linkedinput.java | 44 +- .../examples/injava/NB_start_stop_diag.java | 13 +- .../nbr/examples/injava/NB_threadchange.java | 4 +- nbr/src/main/resources/verify.md | 164 ++ 184 files changed, 4052 insertions(+), 2209 deletions(-) create mode 100644 nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/validators/Cqld4SingleRowValidator.java create mode 100644 nb-adapters/adapter-cqld4/src/main/resources/cqlverify.md create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplateProps.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Validator.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ValidatorSource.java rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{EmitterCycleOp.java => AssertingOp.java} (66%) create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOpDispenser.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOp.java rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{EmitterOpDispenserWrapper.java => CapturingOpDispenser.java} (62%) rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{DryCycleOp.java => DryrunOp.java} (90%) rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{DryCycleOpDispenserWrapper.java => DryrunOpDispenser.java} (90%) rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{EmitterOp.java => ResultPrintingOp.java} (90%) rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/{EmitterCycleOpDispenserWrapper.java => ResultPrintingOpDispenser.java} (86%) create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/DiffType.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/MapDifferencer.java create mode 100644 nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/package-info.java rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/00_workload_specification.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/01_spectest_formatting.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/02_workload_structure.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/04_op_template_basics.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/05_op_template_payloads.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/06_op_template_variations.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/07_template_variables.md (100%) rename nb-apis/adapters-api/src/main/{resources => java}/workload_definition/08_parsed_op_api.md (100%) create mode 100644 nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpAssertions.java create mode 100644 nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpCapture.java rename nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/{OpWrappers.java => OpDryrun.java} (72%) create mode 100644 nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpFunctionComposition.java create mode 100644 nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/ResultValidatingDispenser.java create mode 100644 nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointException.java create mode 100644 nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePoints.java rename nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/VariableCapture.java => nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/DynamicVariableCapture.java (59%) create mode 100644 nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/UniformVariableCapture.java create mode 100644 nbr/src/main/resources/verify.md diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index eed1e26fe..31c1002e2 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -835,6 +835,22 @@ nosqlbench.io https://nosqlbench.io/ + + Madhavan Sridharan + nosqlbench.io + https://nosqlbench.io/ + + + Dave Fisher + nosqlbench.io + https://nosqlbench.io/ + + + Mark Wolters + nosqlbench.io + https://nosqlbench.io/ + + diff --git a/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java b/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java index 7c1fe2c41..c50704df8 100644 --- a/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java +++ b/nb-adapters/adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpOpMapper.java @@ -43,7 +43,7 @@ public class AmqpOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { //public OpDispenser apply(ParsedOp op, LongFunction spaceInitF) { int spaceName = op.getStaticConfigOr("space", 0); diff --git a/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchOpMapper.java b/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchOpMapper.java index 026c084d7..a505e2a40 100644 --- a/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchOpMapper.java +++ b/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchOpMapper.java @@ -57,12 +57,12 @@ public class AzureAISearchOpMapper implements OpMapper, * @param adapterC * @param op * The {@link ParsedOp} to be evaluated - * @param spaceInitF + * @param spaceF * @return The correct {@link AzureAISearchBaseOpDispenser} subclass based on * the op type */ @Override - public OpDispenser> apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser> apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { TypeAndTarget typeAndTarget = op.getTypeAndTarget(AzureAISearchOpType.class, String.class, "type", "target"); diff --git a/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchSpace.java b/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchSpace.java index 512acf578..15aad8c92 100644 --- a/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchSpace.java +++ b/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/AzureAISearchSpace.java @@ -79,21 +79,23 @@ public class AzureAISearchSpace extends BaseSpace { private SearchIndexClient createSearchClients() { String uri = cfg.get("endpoint"); - var requiredToken = cfg.getOptional("token_file").map(Paths::get).map(tokenFilePath -> { - try { - return Files.readAllLines(tokenFilePath).getFirst(); - } catch (IOException e) { - String error = "Error while reading token from file:" + tokenFilePath; - logger.error(error, e); - throw new RuntimeException(e); - } - }).orElseGet(() -> cfg.getOptional("token").orElseThrow(() -> new RuntimeException( - "You must provide either a 'token_file' or a 'token' to configure a Azure AI Search client"))); + String requiredToken = cfg.getOptional("token_file").map(Paths::get).map(tokenFilePath -> { + try { + return Files.readAllLines(tokenFilePath).getFirst(); + } catch (IOException e) { + String error = "Error while reading token from file:" + tokenFilePath; + logger.error(error, e); + throw new RuntimeException(e); + } + }).orElseGet(() -> cfg.getOptional("token").orElse(null)); +// .orElseThrow(() -> new RuntimeException( +// "You must provide either a 'token_file' or a 'token' to configure a Azure AI Search client"))); logger.info(() -> "Creating new Azure AI Search Client with (masked) token/key [" + AzureAISearchAdapterUtils.maskDigits(requiredToken) + "], uri/endpoint [" + uri + "]"); - var searchIndexClientBuilder = new SearchIndexClientBuilder().endpoint(uri); + SearchIndexClientBuilder searchIndexClientBuilder = new SearchIndexClientBuilder().endpoint( + uri); if (!requiredToken.isBlank()) { searchIndexClientBuilder = searchIndexClientBuilder.credential(new AzureKeyCredential(requiredToken)); } else { diff --git a/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/opsdispenser/AzureAISearchUploadDocumentsOpDispenser.java b/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/opsdispenser/AzureAISearchUploadDocumentsOpDispenser.java index 60e5f0f36..e309fd892 100644 --- a/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/opsdispenser/AzureAISearchUploadDocumentsOpDispenser.java +++ b/nb-adapters/adapter-azure-aisearch/src/main/java/io/nosqlbench/adapter/azureaisearch/opsdispenser/AzureAISearchUploadDocumentsOpDispenser.java @@ -66,7 +66,7 @@ public class AzureAISearchUploadDocumentsOpDispenser extends AzureAISearchBaseOp private LongFunction buildFieldsStruct(LongFunction fieldsFunction) { return l -> { Map fields = fieldsFunction.apply(l); - var doc = new SearchDocument(); + SearchDocument doc = new SearchDocument(); fields.forEach((key, val) -> { doc.put(key, val); }); diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java index d7a4cc85f..afed2d6f4 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java @@ -18,6 +18,9 @@ package io.nosqlbench.adapter.cqld4; import io.nosqlbench.adapter.cqld4.opmappers.Cqld4CoreOpMapper; import io.nosqlbench.adapter.cqld4.optypes.Cqld4BaseOp; +import io.nosqlbench.adapter.cqld4.validators.Cqld4SingleRowValidator; +import io.nosqlbench.adapters.api.activityimpl.uniform.Validator; +import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.nb.api.config.standard.NBConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.adapters.api.activityimpl.OpMapper; @@ -29,10 +32,7 @@ import io.nosqlbench.nb.annotations.Service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Function; import java.util.function.LongFunction; diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4BatchStmtMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4BatchStmtMapper.java index 1a890cf67..0e84e517c 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4BatchStmtMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4BatchStmtMapper.java @@ -41,10 +41,10 @@ public class CqlD4BatchStmtMapper> extends Cq } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { ParsedOp subop = op.getAsSubOp("op_template", ParsedOp.SubOpNaming.ParentAndSubKey); int repeat = op.getStaticValue("repeat"); - OpDispenser od = new Cqld4CqlOpMapper(adapter).apply(adapterC, op, spaceInitF); + OpDispenser od = new Cqld4CqlOpMapper(adapter).apply(adapterC, op, spaceF); return new CqlD4BatchStmtDispenser(adapter, op, repeat, subop, od); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4CqlSimpleStmtMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4CqlSimpleStmtMapper.java index 4b82953b6..1721e8a8f 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4CqlSimpleStmtMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4CqlSimpleStmtMapper.java @@ -36,7 +36,7 @@ public class CqlD4CqlSimpleStmtMapper extends Cqld4CqlBaseOpMapper apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { return new Cqld4SimpleCqlStmtDispenser(adapter, targetFunction, op); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4OpType.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4OpType.java index 8a1ec6cc8..f3c06942e 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4OpType.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4OpType.java @@ -18,65 +18,61 @@ package io.nosqlbench.adapter.cqld4.opmappers; public enum CqlD4OpType { - /** - * uses {@link com.datastax.oss.driver.api.core.cql.SimpleStatement} - * does not parameterize values via the SimpleStatement API. - * Pre-renderes the statement string with values included. This is not - * efficient nor recommended for production use, although it is useful - * for certain testing scenarios in which you need to create a lot - * of DDL or other statements which require non-parameterizable fields - * to be present in binding values. - */ + // uses [[com.datastax.oss.driver.api.core.cql.SimpleStatement]] + // **does not** parameterize values via the SimpleStatement API. + // Pre-renderes the statement string with values included. This is not + // efficient nor recommended for production use, although it is useful + // for certain testing scenarios in which you need to create a lot + // of DDL or other statements which require non-parameterizable fields + // to be present in binding values. raw, - /** - * uses {@link com.datastax.oss.driver.api.core.cql.SimpleStatement} - * This parameterizes values and applies them as positional fields, - * where the binding points are aligned by binding name. - */ + // uses [[com.datastax.oss.driver.api.core.cql.SimpleStatement]] + // This parameterizes values and applies them as positional fields, + // where the binding points are aligned by binding name. + // simple, - /** - * uses {@link com.datastax.oss.driver.api.core.cql.SimpleStatement} - * This type does everything that the {@link #simple} mode does, and - * additionally uses prepared statements. - */ + // uses [[com.datastax.oss.driver.api.core.cql.PreparedStatement]] + // This type does everything that the {@link #simple} mode does, and + // additionally uses prepared statements. prepared, - /** - * Allows for a statement template to be used to create a batch statement. - * The fields 'op_template', and 'repeat' are required, and all fields below - * the op_template field are a nested version of the other op types here, but - * supports only the simple and prepared forms for historic compatibility reasons. - */ + // uses [[com.datastax.oss.driver.api.core.cql.BatchStatement]] + // Allows for a statement template to be used to create a batch statement. + // The fields 'op_template', and 'repeat' are required, and all fields below + // the op_template field are a nested version of the other op types here, but + // supports only the simple and prepared forms for historic compatibility reasons. batch, - /** - * uses {@link com.datastax.dse.driver.api.core.graph.ScriptGraphStatement} - * This is the "raw" mode of using gremlin. It is not as efficient, and thus - * is only recommended for testing or legacy apps. - */ + // This reads rows of data and verifies data and structure according to what + // would have been written according to the original bindings. + verify, + + // uses [[com.datastax.dse.driver.api.core.graph.ScriptGraphStatement]] + // This is the "raw" mode of using gremlin. It is not as efficient, and thus + // is only recommended for testing or legacy apps. gremlin, /** - * uses {@link com.datastax.dse.driver.api.core.graph.FluentGraphStatement} + * uses [[com.datastax.dse.driver.api.core.graph.FluentGraphStatement]] * This mode is the recommended mode for gremlin execution. It uses the fluent * API on the client side. The fluent syntax is compiled and cached as bytecode * within a per-thread execution environment (for each op template). For each * cycle, the bindings are rendered, injected into that execution environment, * and then the bytecode is executed to render the current operation, which is * then sent to the server. Although this is arguably more involved, the result - * is quite efficient and provides the closes idiomatic experience AND + * is quite efficient and provides the closes idiomatic experience **AND** * the best performance. * - *

This is the mode that is recommended for all graph usage.

+ * This is the mode that is recommended for all graph usage. */ fluent, - /** - * reserved for future use - */ - rainbow, +// /** +// * reserved for future use +// */ +// rainbow, // /** // * reserved for future use // */ diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4PreparedStmtMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4PreparedStmtMapper.java index b09485fea..6354d8601 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4PreparedStmtMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4PreparedStmtMapper.java @@ -47,7 +47,7 @@ public class CqlD4PreparedStmtMapper extends Cqld4CqlBaseOpMapper apply( NBComponent adapterC, ParsedOp op, - LongFunction spaceInitF + LongFunction spaceF ) { ParsedTemplateString stmtTpl = op.getAsTemplate(target.field).orElseThrow(() -> new BasicError( "No statement was found in the op template:" + op @@ -68,7 +68,7 @@ public class CqlD4PreparedStmtMapper extends Cqld4CqlBaseOpMapper extends Cql } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { return null; // return new CqlD4RainbowTableDispenser(adapter, sessionFunc,targetFunction, op); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4RawStmtMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4RawStmtMapper.java index fdba78fb7..8c9097a0b 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4RawStmtMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4RawStmtMapper.java @@ -35,7 +35,7 @@ public class CqlD4RawStmtMapper extends Cqld4CqlBaseOpMapper apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { return new Cqld4RawStmtDispenser(adapter, targetFunction,op); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4BaseOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4BaseOpMapper.java index 483f15f23..148a6af3e 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4BaseOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4BaseOpMapper.java @@ -38,5 +38,5 @@ public abstract class Cqld4BaseOpMapper> implements OpM } @Override - public abstract OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF); + public abstract OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java index 698898ef3..afa842bd6 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CoreOpMapper.java @@ -61,8 +61,8 @@ public class Cqld4CoreOpMapper extends Cqld4BaseOpMapper> { case raw, simple, prepared, batch -> new Cqld4CqlOpMapper(adapter).apply(adapterC, op, spaceF); case gremlin -> new Cqld4GremlinOpMapper(adapter, target.targetFunction).apply(adapterC, op, spaceF); case fluent -> new Cqld4FluentGraphOpMapper(adapter, target).apply(adapterC, op, spaceF); - case rainbow -> - new CqlD4RainbowTableMapper(adapter, spaceF, target.targetFunction).apply(adapterC, op, spaceF); +// case rainbow -> +// new CqlD4RainbowTableMapper(adapter, spaceF, target.targetFunction).apply(adapterC, op, spaceF); default -> throw new OpConfigError("Unsupported op type " + opType); // case sst -> new Cqld4SsTableMapper(adapter, sessionFunc, target.targetFunction).apply(op); }; diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlBaseOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlBaseOpMapper.java index eba4a9989..77b3c3d47 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlBaseOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlBaseOpMapper.java @@ -34,6 +34,6 @@ public abstract class Cqld4CqlBaseOpMapper extends Cqld4Ba } @Override - public abstract OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF); + public abstract OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF); } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlOpMapper.java index ae137b10f..736d97969 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4CqlOpMapper.java @@ -39,7 +39,7 @@ public class Cqld4CqlOpMapper extends Cqld4CqlBaseOpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { CqlD4OpType opType = CqlD4OpType.prepared; TypeAndTarget target = op.getTypeAndTarget(CqlD4OpType.class, String.class, "type", "stmt"); logger.info(() -> "Using " + target.enumId + " statement form for '" + op.getName() + "'"); @@ -47,13 +47,13 @@ public class Cqld4CqlOpMapper extends Cqld4CqlBaseOpMapper { return (OpDispenser) switch (target.enumId) { case raw -> { CqlD4RawStmtMapper cqlD4RawStmtMapper = new CqlD4RawStmtMapper(adapter, target.targetFunction); - OpDispenser apply = cqlD4RawStmtMapper.apply(adapterC, op, spaceInitF); + OpDispenser apply = cqlD4RawStmtMapper.apply(adapterC, op, spaceF); yield apply; } - case simple -> new CqlD4CqlSimpleStmtMapper(adapter, target.targetFunction).apply(adapterC, op, spaceInitF); - case prepared -> new CqlD4PreparedStmtMapper(adapter, target).apply(adapterC, op, spaceInitF); + case simple -> new CqlD4CqlSimpleStmtMapper(adapter, target.targetFunction).apply(adapterC, op, spaceF); + case prepared -> new CqlD4PreparedStmtMapper(adapter, target).apply(adapterC, op, spaceF); - case batch -> new CqlD4BatchStmtMapper(adapter, target).apply(adapterC, op, spaceInitF); + case batch -> new CqlD4BatchStmtMapper(adapter, target).apply(adapterC, op, spaceF); default -> throw new OpConfigError("Unsupported op type for CQL category of statement forms:" + target.enumId); }; diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4FluentGraphOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4FluentGraphOpMapper.java index 26e793282..dce0eef81 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4FluentGraphOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4FluentGraphOpMapper.java @@ -59,7 +59,7 @@ public class Cqld4FluentGraphOpMapper extends Cqld4BaseOpMapper apply(NBComponent adapterC, ParsedOp op, LongFunction cqld4SpaceLongFunction) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { GraphTraversalSource g = DseGraph.g; ParsedTemplateString fluent = op.getAsTemplate(target.field).orElseThrow(); diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4GremlinOpMapper.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4GremlinOpMapper.java index 72eaba185..a259cbfd1 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4GremlinOpMapper.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4GremlinOpMapper.java @@ -33,7 +33,7 @@ public class Cqld4GremlinOpMapper extends Cqld4Ba } @Override - public Cqld4GremlinOpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public Cqld4GremlinOpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { return new Cqld4GremlinOpDispenser( adapter, l -> adapter.getSpaceFunc(op).apply(l).getSession(), targetFunction, op); diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java index 801e0c779..de5ced85f 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.cqld4.optypes; +import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.cql.*; import io.nosqlbench.adapter.cqld4.Cqld4CqlReboundStatement; @@ -26,14 +27,16 @@ import io.nosqlbench.adapter.cqld4.exceptions.ExceededRetryReplaceException; import io.nosqlbench.adapter.cqld4.exceptions.UnexpectedPagingException; import io.nosqlbench.adapter.cqld4.instruments.CqlOpMetrics; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.*; -import org.apache.commons.lang3.NotImplementedException; +import io.nosqlbench.virtdata.core.templates.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.*; +import java.util.function.Function; // TODO: add statement filtering @@ -47,7 +50,7 @@ import java.util.concurrent.*; public abstract class Cqld4CqlOp - implements Cqld4BaseOp>, VariableCapture, OpGenerator, OpResultSize { + implements Cqld4BaseOp>, UniformVariableCapture>, OpGenerator, OpResultSize { private final static Logger logger = LogManager.getLogger(Cqld4CqlOp.class); @@ -165,8 +168,33 @@ public abstract class Cqld4CqlOp return next; } - public Map capture() { - throw new NotImplementedException("Not implemented for Cqld4CqlOp"); + @Override + public Function, Map> initCaptureF(CapturePoints> points) { + Function,Map> f = (List result) -> { + if (result.size()!=1) { + throw new CapturePointException("result contained " + result.size() + " rows, required exactly 1"); + } + Row row = result.get(0); + ColumnDefinitions coldefs = row.getColumnDefinitions(); + Map values = new HashMap<>(coldefs.size()); + + if (points.isGlob()) { + for (ColumnDefinition coldef : coldefs) { + String colname = coldef.getName().toString(); + values.put(colname,row.getObject(colname)); + } + } else { + for (CapturePoint> point : points) { + String sourceName = point.getSourceName(); + Object value = row.getObject(point.getSourceName()); + Object recast = point.getAsCast().cast(value); + values.put(point.getAsName(), recast); + } + } + + return values; + }; + return f; } public abstract Statement getStmt(); diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4RainbowTableOp.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4RainbowTableOp.java index 129aad489..7c6f369dc 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4RainbowTableOp.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4RainbowTableOp.java @@ -19,11 +19,8 @@ package io.nosqlbench.adapter.cqld4.optypes; import com.datastax.oss.driver.api.core.cql.ResultSet; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.*; -import java.util.Map; - // Need to create RainbowTableStatement -public class Cqld4RainbowTableOp - implements Cqld4BaseOp, VariableCapture, OpGenerator, OpResultSize { +public class Cqld4RainbowTableOp implements Cqld4BaseOp, OpGenerator, OpResultSize { // private final CqlSession session; // private final RainbowTableStatement stmt; @@ -39,9 +36,4 @@ public class Cqld4RainbowTableOp throw new RuntimeException("implement me"); } - @Override - public Map capture() { - throw new RuntimeException("implement me"); - } - } diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/validators/Cqld4SingleRowValidator.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/validators/Cqld4SingleRowValidator.java new file mode 100644 index 000000000..008072a78 --- /dev/null +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/validators/Cqld4SingleRowValidator.java @@ -0,0 +1,35 @@ +package io.nosqlbench.adapter.cqld4.validators; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import com.datastax.oss.driver.api.core.cql.Row; +import io.nosqlbench.adapters.api.activityimpl.uniform.Validator; + +import java.util.List; + +public class Cqld4SingleRowValidator implements Validator> { + + public Cqld4SingleRowValidator() { + } + + @Override + public void validate(List rows) { + System.out.println("validating rows..."); + } +} diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java index 19e864b04..0a7163c0b 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java @@ -536,7 +536,7 @@ public class CGWorkloadExporter implements BundledApp { pkeys.pop(); } } - var lastcount = keycount; + int lastcount = keycount; keycount = Math.max(table.getPartitionKeys().size(), keycount); if (keycount != lastcount) { logger.debug("minimum keycount for " + table.getFullName() + " adjusted from " + lastcount + " to " + keycount); diff --git a/nb-adapters/adapter-cqld4/src/main/resources/cqlverify.md b/nb-adapters/adapter-cqld4/src/main/resources/cqlverify.md new file mode 100644 index 000000000..b1de99e50 --- /dev/null +++ b/nb-adapters/adapter-cqld4/src/main/resources/cqlverify.md @@ -0,0 +1,162 @@ +# CQL Verify + +The cqld4 adapter has the capability to verify data in a database by reading each row and comparing +it with reference data. The reference data is simply the same data which you might use to generate +data for upsert, modeled with standard bindings. Thus, you can assert the correctness of any (whole +or part of) data within a database for which you have data bindings. For the most common case -- +writing and then verifying data, the same bindings are used for both phases. + +It allows you to read values from a database and compare them to the generated values that were +expected to be written, row-by-row, producing a comparative result between the two. + +If you specify `verify` as the op type with the cqld4 adapter, then it will do the following: + +1. Presume that the op structure is effectively a read operation, meaning it will produce a result + that can be used for comparison. The operation must have a single row in the result set. If there + is no row, then the row fails validation. The same happens if there is more than one row. +2. Using the provided bindings, re-generate the data separately which was expected to be in the + database. This is called _reference data_ herein. +3. Verify that the values returned from the database are the same as the reference data, and throw a + ResultVerificationError for each mis-matching row. + +Alternately, you can add the verify and compare options to any regular CQL statement (Simple, Raw, +Prepared) in order to enable this verification logic. + +## Verification Options + +These options may be attached to an op template directly as op fields, or they may be passed as +activity parameters. NOTE: passing them as activity parameters will only work if all of the active +operations are compatible with the verify options. + +- **verify** - an optional modifier of fields to verify for a statement. If this parameter is not + provided, then it is presumed to be `*` by default. This is a string which consists of + comma-separate values. This parameter is useful if you have a set of default bindings and want to + specify which subset of them will be used just for this statement. Each form modifies the list of + fields to verify incrementally, and multiple options are allowed: +- `*` : If the value is `*`, then all the bindings that are visible for the statement will be used + as expected values. +- `-`, `-field2` : If it is a word that starts with `-`, like `-field2`, then the name after the + dash is removed from the list of fields to verify. +- `+`, `+field3` : If it is a word that starts with a `+`, like `+field3`, or a simple word, then + the field is added to the list of fields to verify. +- `f->b` : If any of the added fields is in the form `f->b`, then it is taken as a mapping from the + field name `f` in the schema to a binding `b`. + +For example, + +```yaml +# example op template +ops: + op1: + readit: "select ....TBD" + bindings: + a: .. + b: .. + c: .. + verify: "*,-b" +``` + +means _verify all fields from the bindings except `b`_, using the default validation method. + +- **compare** - what to verify, naming structure, values, etc. each of these is additive, and + multiple can be specified. + - all (the default) - A synonym for fields AND values + - fields - A synonym for rowfields AND reffields (full set equivalence) + - rowfields - Verify that the result field names include every reference field name. + - reffields - Verify that the reference field names include every result field name. + - values - Verify that all the pair-wise fields have equal values, according to the + type-specific `.equals(...)` method for the data type identified in the row metadata by field + name. + +For example, + +```yaml +# example op template +ops: + op1: + readit: "select ....TBD" + bindings: + a: .. + b: .. + c: .. + verify: "*,-b" + compare: reffields +``` + +means _ensure reference fields are present in result fields by name but do not throw an error if +there are more result fields present, and do not compare values of same-named fields_. + +## Verification Results + +### Errors + +The data bindings are used to generate the expected values that would be used for an upsert. Each +row is verified according to these values, and any discrepancy is treated as an error that can be +counted, logged, etc. If you want to simply count the occurences instead of fast-failing an activity +when a row is unverified, then you simply need to modify the error handler for that activity: ' +errors=unverified=count'. + +The default error handling behavior will cause an exception to be thrown and NoSQLBench +will exit by default. If you wish for something less dramatic, then +wire the unverified category to something else: + + errors=...,unverified->count + +or + + errors=...,unverified->warn + +### Metrics + +The cqlverify activity type adds some verification-specific metrics: + +- alias.verified_results - A counter for how many results were verified +- alias.unverified_results - A counter for how many results were not verified +- alias.verified_values - A counter for how many field values were verified +- alias.unverified_values - A counter for how many field values were unverified + +## Advanced Usage + +The verify capability does not retain logged data for verification. Still, it is able to compare +data as if it had a separate physical data set to compare to. This is possible only because +virtdata (the data generation layer of NoSQLBench) can provide realistic views of virtual datasets +on the fly. + +### Avoid Random data + +That means, however, that you must avoid using the non-stable data mapping functions when writing +data. The rule of thumb is to avoid using any data mapping functions containing the word "Random". +Binding functions with `random` in their name behave differently from others in that they will not +produce stable results. Their initialization vector is external to the function definition, such as +when using the system _random_ functions. + +> Some will bristle at this misuse of the terms, but connotatively they work well for most +> other users. Actually, all of the algorithms used by +> virtdata are __NOT__ truly random, and are deterministic in some way. However, some rely on +> an initialization vector which is not self-contained within the function definition. As such, +> these functions are not pure functions in practice and thus may not be relied upon to return the +> same result from session to session. The word _random_ in virtdata binding functions indicates +> that a function is non-determinstic in some cases. As long as you avoid these functions, +> you can rely on stable generated data from session to session. + +### Single vs Multiple Results + +TBD + +### Paging vs Non-Paging verification + +TBD + +### Example activity definitions + +Write 100K cycles of telemetry data + +> TBD ... run driver=cql alias=writesome workload=cql-iot tags=group:write cycles=100000 host=... + +Verify the the same 100K cycles of telemetry data + +> TBD ... run driver=cqlverify alias=verify workload=cql-iot tags=group:verify cycles=100000 + +To see how these examples work, TBD + + diff --git a/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java b/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java index a5818743d..5f05e8504 100644 --- a/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java +++ b/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java @@ -52,7 +52,7 @@ public class CqlParserHarnessTest { CGWorkloadExporter exporter = new CGWorkloadExporter(); exporter.applyAsInt(new String[]{"src/test/resources/testschemas/cql_alltypes.cql","_alltypes.yaml"}); exporter.setNamingTemplate("[OPTYPE-][COLUMN-][TYPEDEF-][TABLE-]-[KEYSPACE]"); - var data = exporter.getWorkloadAsYaml(); + String data = exporter.getWorkloadAsYaml(); } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 8ef6dbd82..ffbb771ea 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -38,7 +38,7 @@ public class DataApiOpMapper implements OpMapper { @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { //public OpDispenser apply(ParsedOp op, LongFunction spaceInitF) { TypeAndTarget typeAndTarget = op.getTypeAndTarget( DataApiOpType.class, diff --git a/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java b/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java index e24066747..f7691ab51 100644 --- a/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java +++ b/nb-adapters/adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBOpMapper.java @@ -40,7 +40,7 @@ public class DynamoDBOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { int space = op.getStaticConfigOr("space", 0); LongFunction spaceFunc = adapter.getSpaceFunc(op); DynamoDB ddb = spaceFunc.apply(space).getDynamoDB(); diff --git a/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleOpMapper.java b/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleOpMapper.java index 80f8e4fa1..764a0c760 100644 --- a/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleOpMapper.java +++ b/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleOpMapper.java @@ -2,13 +2,13 @@ package io.nosqlbench.adapter.prototype; /* * Copyright (c) nosqlbench - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -35,13 +35,13 @@ public class ExampleOpMapper implements OpMapper { public OpDispenser apply( NBComponent adapterC, ParsedOp pop, - LongFunction spaceInitF + LongFunction spaceF ) { TypeAndTarget typeAndTarget = pop.getTypeAndTarget(ExampleOpTypes.class, String.class); return switch (typeAndTarget.enumId) { - case type1 -> new ExampleOpDispenserType1(adapterC, pop, spaceInitF); - case type2 -> new ExampleOpDispenserType1(adapterC, pop, spaceInitF); + case type1 -> new ExampleOpDispenserType1(adapterC, pop, spaceF); + case type2 -> new ExampleOpDispenserType1(adapterC, pop, spaceF); }; } diff --git a/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleSpace.java b/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleSpace.java index 19f888286..07d04c409 100644 --- a/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleSpace.java +++ b/nb-adapters/adapter-example/src/main/java/io/nosqlbench/adapter/prototype/ExampleSpace.java @@ -2,13 +2,13 @@ package io.nosqlbench.adapter.prototype; /* * Copyright (c) nosqlbench - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -21,6 +21,9 @@ package io.nosqlbench.adapter.prototype; import io.nosqlbench.adapters.api.activityimpl.uniform.BaseSpace; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +/** + * {@inheritDoc} + */ public class ExampleSpace extends BaseSpace { public ExampleSpace(DriverAdapter adapter, long idx) { diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index 62775487a..9ec91daf5 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -51,12 +51,12 @@ public class GCPSpannerOpMapper implements OpMapper, GCPSp * @param adapterC * @param op * The {@link ParsedOp} to be evaluated - * @param spaceInitF + * @param spaceF * @return The correct {@link GCPSpannerBaseOpDispenser} subclass based on * the op type */ @Override - public OpDispenser> apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser> apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { TypeAndTarget typeAndTarget = op.getTypeAndTarget(GCPSpannerOpType.class, String.class, "type", "target"); logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java index 5223338a1..83dec2780 100644 --- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java +++ b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpMapper.java @@ -37,8 +37,8 @@ public class HttpOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { LongFunction spaceNameF = op.getAsFunctionOr("space", "default"); - return new HttpOpDispenser(adapter, spaceInitF, op); + return new HttpOpDispenser(adapter, spaceF, op); } } diff --git a/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java b/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java index b20c5d03a..3c70ba9e9 100644 --- a/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java +++ b/nb-adapters/adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaOpMapper.java @@ -42,7 +42,7 @@ public class KafkaOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { KafkaSpace kafkaSpace = adapter.getSpaceFunc(op).apply(op.getStaticConfigOr("space",0)); /* diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusDriverAdapter.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusDriverAdapter.java index b7bccc055..5ebcd7c4b 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusDriverAdapter.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusDriverAdapter.java @@ -27,6 +27,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.nb.api.labels.NBLabels; import java.util.function.Function; +import java.util.function.LongFunction; import static io.nosqlbench.adapter.milvus.MilvusAdapterUtils.MILVUS; @@ -38,15 +39,20 @@ public class MilvusDriverAdapter extends BaseDriverAdapter, Milv } @Override - public OpMapper> getOpMapper() { + public OpMapper,MilvusSpace> getOpMapper() { return new MilvusOpMapper(this); } @Override - public Function getSpaceInitializer(NBConfiguration cfg) { - return (s) -> new MilvusSpace(s, cfg); + public LongFunction getSpaceInitializer(NBConfiguration cfg) { + return (idx) -> new MilvusSpace(this, idx, cfg); } + // @Override +// public Function getSpaceInitializer(NBConfiguration cfg) { +// return (s) -> new MilvusSpace(s, cfg); +// } + @Override public NBConfigModel getConfigModel() { return super.getConfigModel().add(MilvusSpace.getConfigModel()); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusOpMapper.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusOpMapper.java index b670e15a2..4f2bd71ee 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusOpMapper.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusOpMapper.java @@ -23,110 +23,157 @@ import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.engine.api.templating.TypeAndTarget; +import io.nosqlbench.nb.api.components.core.NBComponent; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class MilvusOpMapper implements OpMapper> { +import java.util.function.LongFunction; + +public class MilvusOpMapper implements OpMapper, MilvusSpace> { private static final Logger logger = LogManager.getLogger(MilvusOpMapper.class); private final MilvusDriverAdapter adapter; /** - * Create a new MilvusOpMapper implementing the {@link OpMapper} interface. - * - * @param adapter The associated {@link MilvusDriverAdapter} + Create a new MilvusOpMapper implementing the {@link OpMapper} interface. + @param adapter + The associated {@link MilvusDriverAdapter} */ public MilvusOpMapper(MilvusDriverAdapter adapter) { this.adapter = adapter; } /** - * Given an instance of a {@link ParsedOp} returns the appropriate {@link MilvusBaseOpDispenser} subclass - * - * @param op The {@link ParsedOp} to be evaluated - * @return The correct {@link MilvusBaseOpDispenser} subclass based on the op type + Given an instance of a {@link ParsedOp} returns the appropriate {@link MilvusBaseOpDispenser} + subclass + @param op + The {@link ParsedOp} to be evaluated + @return The correct {@link MilvusBaseOpDispenser} subclass based on the op type */ @Override - public OpDispenser> apply(ParsedOp op) { + public OpDispenser> apply( + NBComponent adapterC, ParsedOp op, + LongFunction spaceF + ) { + TypeAndTarget typeAndTarget = op.getTypeAndTarget( MilvusOpType.class, String.class, - "type", - "target" + "type", "target" ); - logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); + logger.info( + () -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); return switch (typeAndTarget.enumId) { - case drop_collection -> new MilvusDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_collection -> new MilvusCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_index -> new MilvusCreateIndexOpDispenser(adapter, op, typeAndTarget.targetFunction); - case drop_index -> new MilvusDropIndexOpDispenser(adapter, op, typeAndTarget.targetFunction); + case drop_collection -> new MilvusDropCollectionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case create_collection -> new MilvusCreateCollectionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case create_index -> new MilvusCreateIndexOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case drop_index -> new MilvusDropIndexOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); // Uses the Collection-specific fields (columnar) insert mode - case insert_rows -> new MilvusInsertRowsOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert_rows -> new MilvusInsertRowsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); // Uses the High-Level row-by-row JSONObject (tabular) insert mode - case insert -> new MilvusInsertOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete -> new MilvusDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction); - case search -> new MilvusSearchOpDispenser(adapter, op, typeAndTarget.targetFunction); - case alter_alias -> new MilvusAlterAliasOpDispenser(adapter, op, typeAndTarget.targetFunction); - case alter_collection -> new MilvusAlterCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case flush -> new MilvusFlushOpDispenser(adapter, op, typeAndTarget.targetFunction); - case bulk_insert -> new MilvusBulkInsertOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_alias -> new MilvusCreateAliasOpDispenser(adapter, op, typeAndTarget.targetFunction); - case get -> new MilvusGetOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_partition -> new MilvusCreatePartitionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_credential -> new MilvusCreateCredentialOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_database -> new MilvusCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); - case query -> new MilvusQueryOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete_credential -> new MilvusDeleteCredentialOpDispenser(adapter, op, typeAndTarget.targetFunction); - case describe_collection -> - new MilvusDescribeCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case describe_index -> new MilvusDescribeIndexOpDispenser(adapter, op, typeAndTarget.targetFunction); - case drop_alias -> new MilvusDropAliasOpDispenser(adapter, op, typeAndTarget.targetFunction); - case get_metrics -> new MilvusGetMetricsOpDispenser(adapter, op, typeAndTarget.targetFunction); - case drop_database -> new MilvusDropDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); - case get_replicas -> new MilvusGetReplicasOpDispenser(adapter, op, typeAndTarget.targetFunction); - case load_balance -> new MilvusLoadBalanceOpDispenser(adapter, op, typeAndTarget.targetFunction); - case has_partition -> new MilvusHasPartitionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case drop_partition -> new MilvusDropPartitionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case get_load_state -> new MilvusGetLoadStateOpDispenser(adapter, op, typeAndTarget.targetFunction); - case list_databases -> new MilvusListDatabasesOpDispenser(adapter, op, typeAndTarget.targetFunction); - case manual_compact -> new MilvusManualCompactOpDispenser(adapter, op, typeAndTarget.targetFunction); - case get_index_state -> new MilvusGetIndexStateOpDispenser(adapter, op, typeAndTarget.targetFunction); - case list_cred_users -> new MilvusListCredUsersOpDispenser(adapter, op, typeAndTarget.targetFunction); - case load_collection -> new MilvusLoadCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case show_partitions -> new MilvusShowPartitionsOpDispenser(adapter, op, typeAndTarget.targetFunction); - case load_partitions -> new MilvusLoadPartitionsOpDispenser(adapter, op, typeAndTarget.targetFunction); - case list_collections -> new MilvusListCollectionsOpDispenser(adapter, op, typeAndTarget.targetFunction); - case show_collections -> new MilvusShowCollectionsOpDispenser(adapter, op, typeAndTarget.targetFunction); - case update_credential -> new MilvusUpdateCredentialOpDispenser(adapter, op, typeAndTarget.targetFunction); - case release_collection -> new MilvusReleaseCollectionOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_bulk_insert_state -> new MilvusGetBulkInsertStateOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case release_partitions -> new MilvusReleasePartitionsOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_flush_all_state -> new MilvusGetFlushAllStateOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_compaction_state -> new MilvusGetCompactionStateOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_loading_progress -> new MilvusGetLoadingProgressOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_persistent_segment_info -> new MilvusGetPersistentSegmentInfoOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_query_segment_info -> new MilvusGetQuerySegmentInfoOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case list_bulk_insert_tasks -> new MilvusListBulkInsertTasksOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_index_build_progress -> new MilvusGetIndexBuildProgressOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_partition_statistics -> new MilvusGetPartitionStatisticsOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_collection_statistics -> new MilvusGetCollectionStatisticsOpDispenser(adapter, op, - typeAndTarget.targetFunction); - case get_compaction_state_with_plans -> new MilvusGetCompactionStateWithPlansOpDispenser(adapter, op, - typeAndTarget.targetFunction); + case insert -> new MilvusInsertOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case delete -> new MilvusDeleteOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case search -> new MilvusSearchOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case alter_alias -> new MilvusAlterAliasOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case alter_collection -> new MilvusAlterCollectionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case flush -> new MilvusFlushOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case bulk_insert -> new MilvusBulkInsertOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case create_alias -> new MilvusCreateAliasOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get -> new MilvusGetOpDispenser(adapter, op, typeAndTarget.targetFunction, spaceF); + case create_partition -> new MilvusCreatePartitionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case create_credential -> new MilvusCreateCredentialOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case create_database -> new MilvusCreateDatabaseOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case query -> new MilvusQueryOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case delete_credential -> new MilvusDeleteCredentialOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case describe_collection -> new MilvusDescribeCollectionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case describe_index -> new MilvusDescribeIndexOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case drop_alias -> new MilvusDropAliasOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_metrics -> new MilvusGetMetricsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case drop_database -> new MilvusDropDatabaseOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_replicas -> new MilvusGetReplicasOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case load_balance -> new MilvusLoadBalanceOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case has_partition -> new MilvusHasPartitionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case drop_partition -> new MilvusDropPartitionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_load_state -> new MilvusGetLoadStateOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case list_databases -> new MilvusListDatabasesOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case manual_compact -> new MilvusManualCompactOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_index_state -> new MilvusGetIndexStateOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case list_cred_users -> new MilvusListCredUsersOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case load_collection -> new MilvusLoadCollectionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case show_partitions -> new MilvusShowPartitionsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case load_partitions -> new MilvusLoadPartitionsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case list_collections -> new MilvusListCollectionsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case show_collections -> new MilvusShowCollectionsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case update_credential -> new MilvusUpdateCredentialOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case release_collection -> new MilvusReleaseCollectionOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_bulk_insert_state -> new MilvusGetBulkInsertStateOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case release_partitions -> new MilvusReleasePartitionsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_flush_all_state -> new MilvusGetFlushAllStateOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_compaction_state -> new MilvusGetCompactionStateOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_loading_progress -> new MilvusGetLoadingProgressOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_persistent_segment_info -> new MilvusGetPersistentSegmentInfoOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_query_segment_info -> new MilvusGetQuerySegmentInfoOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case list_bulk_insert_tasks -> new MilvusListBulkInsertTasksOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_index_build_progress -> new MilvusGetIndexBuildProgressOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_partition_statistics -> new MilvusGetPartitionStatisticsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_collection_statistics -> new MilvusGetCollectionStatisticsOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); + case get_compaction_state_with_plans -> + new MilvusGetCompactionStateWithPlansOpDispenser( + adapter, op, typeAndTarget.targetFunction, spaceF); // default -> throw new RuntimeException("Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + // "mapping parsed op " + op); }; } + } diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java index e9dc60776..1482ea474 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java @@ -18,6 +18,8 @@ package io.nosqlbench.adapter.milvus; import io.milvus.client.MilvusServiceClient; import io.milvus.param.ConnectParam; +import io.nosqlbench.adapters.api.activityimpl.uniform.BaseSpace; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.nb.api.config.standard.ConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfiguration; @@ -39,26 +41,16 @@ import java.util.Optional; * https://milvus.io/docs/install-java.md * https://docs.zilliz.com/docs/connect-to-cluster */ -public class MilvusSpace implements AutoCloseable { +public class MilvusSpace extends BaseSpace { private final static Logger logger = LogManager.getLogger(MilvusSpace.class); - private final String name; private final NBConfiguration cfg; protected MilvusServiceClient client; -// private final Map connections = new HashMap<>(); - - /** - * Create a new MilvusSpace Object which stores all stateful contextual information needed to interact - * with the Milvus/Zilliz database instance. - * - * @param name - * The name of this space - * @param cfg - * The configuration ({@link NBConfiguration}) for this nb run - */ - public MilvusSpace(String name, NBConfiguration cfg) { - this.name = name; + public MilvusSpace( + DriverAdapter adapter, long idx, NBConfiguration cfg + ) { + super(adapter, idx); this.cfg = cfg; } @@ -70,7 +62,7 @@ public class MilvusSpace implements AutoCloseable { } private MilvusServiceClient createClient() { - var builder = ConnectParam.newBuilder(); + ConnectParam.Builder builder = ConnectParam.newBuilder(); builder = builder.withUri(cfg.get("uri")); cfg.getOptional("database_name").ifPresent(builder::withDatabaseName); cfg.getOptional("database").ifPresent(builder::withDatabaseName); @@ -95,7 +87,7 @@ public class MilvusSpace implements AutoCloseable { String tokenSummary = Optional.ofNullable(builder.getToken()) .map(MilvusAdapterUtils::maskDigits).orElse("[none]"); logger.info("{}: Creating new Milvus/Zilliz Client with (masked) token [{}], uri/endpoint [{}]", - this.name, tokenSummary, builder.getUri()); + this.getName(), tokenSummary, builder.getUri()); return new MilvusServiceClient(connectParams); } diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterAliasOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterAliasOpDispenser.java index 8b078a89b..951bbfd94 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterAliasOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterAliasOpDispenser.java @@ -17,20 +17,15 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; -import io.milvus.common.clientenum.ConsistencyLevelEnum; import io.milvus.param.alias.AlterAliasParam; -import io.milvus.param.collection.CreateCollectionParam; -import io.milvus.param.collection.FieldType; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusAlterAliasOp; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; -import io.nosqlbench.adapter.milvus.ops.MilvusCreateCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.List; -import java.util.Map; import java.util.function.LongFunction; public class MilvusAlterAliasOpDispenser extends MilvusBaseOpDispenser { @@ -38,8 +33,10 @@ public class MilvusAlterAliasOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -53,7 +50,7 @@ public class MilvusAlterAliasOpDispenser extends MilvusBaseOpDispenser lastF = ebF; final LongFunction collectionParamF = l -> lastF.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterCollectionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterCollectionOpDispenser.java index 8d2c09377..834ebcee1 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterCollectionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusAlterCollectionOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.AlterCollectionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusAlterCollectionOp; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusAlterCollectionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBaseOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBaseOpDispenser.java index ae0f514ae..c4f032626 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBaseOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBaseOpDispenser.java @@ -33,16 +33,15 @@ public abstract class MilvusBaseOpDispenser extends BaseOpDispenser> opF; private final LongFunction paramF; - protected MilvusBaseOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, LongFunction targetF) { - super((DriverAdapter)adapter, op); + protected MilvusBaseOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, + LongFunction targetF, + LongFunction spaceF) { + super(adapter,op,spaceF); this.mzSpaceFunction = adapter.getSpaceFunc(op); this.clientFunction = (long l) -> this.mzSpaceFunction.apply(l).getClient(); this.paramF = getParamFunc(this.clientFunction,op,targetF); this.opF = createOpFunc(paramF, this.clientFunction, op, targetF); } - protected MilvusDriverAdapter getDriverAdapter() { - return (MilvusDriverAdapter) adapter; - } public abstract LongFunction getParamFunc( LongFunction clientF, diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBulkInsertOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBulkInsertOpDispenser.java index d0054f3a4..d8bdcb8b9 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBulkInsertOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusBulkInsertOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.bulkinsert.BulkInsertParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusBulkInsertOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +32,10 @@ public class MilvusBulkInsertOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -53,7 +56,7 @@ public class MilvusBulkInsertOpDispenser extends MilvusBaseOpDispenser finalEbF = ebF; return l -> finalEbF.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateAliasOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateAliasOpDispenser.java index 2d936f054..1d3f9fe48 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateAliasOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateAliasOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.alias.CreateAliasParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusCreateAliasOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusCreateAliasOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -42,7 +45,7 @@ public class MilvusCreateAliasOpDispenser extends MilvusBaseOpDispenser ebF = l -> CreateAliasParam.newBuilder().withAlias(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection",String.class, CreateAliasParam.Builder::withCollectionName); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java index 158edf033..a1e2ad4cc 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java @@ -23,6 +23,7 @@ import io.milvus.param.collection.CollectionSchemaParam; import io.milvus.param.collection.CreateCollectionParam; import io.milvus.param.collection.FieldType; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusCreateCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -38,16 +39,20 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -111,7 +116,7 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser builder.withMaxLength(n.intValue())); fieldspec.getOptionalStaticConfig("max_capacity", Number.class) .ifPresent((Number n) -> builder.withMaxCapacity(n.intValue())); - fieldspec.getOptionalStaticValue(List.of("partition_key", "partition"), Boolean.class) + fieldspec.getOptionalStaticValue("partition", Boolean.class) .ifPresent(builder::withPartitionKey); fieldspec.getOptionalStaticValue("dimension", Number.class) .ifPresent((Number n) -> builder.withDimension(n.intValue())); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCredentialOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCredentialOpDispenser.java index 37580e2af..ea7ebdef2 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCredentialOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCredentialOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.credential.CreateCredentialParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusCreateCredentialOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusCreateCredentialOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateDatabaseOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateDatabaseOpDispenser.java index 4e2afb16d..3a4fc03c7 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateDatabaseOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateDatabaseOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.CreateDatabaseParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusCreateDatabaseOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusCreateDatabaseOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateIndexOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateIndexOpDispenser.java index ea167b100..ec13a899d 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateIndexOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateIndexOpDispenser.java @@ -21,6 +21,7 @@ import io.milvus.param.IndexType; import io.milvus.param.MetricType; import io.milvus.param.index.CreateIndexParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusCreateIndexOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -34,18 +35,20 @@ public class MilvusCreateIndexOpDispenser extends MilvusBaseOpDispenser targetFunction + LongFunction targetFunction, LongFunction spaceF ) { - super(adapter, op, targetFunction); + super(adapter, op, targetFunction,spaceF); } @Override @@ -53,7 +56,7 @@ public class MilvusCreateIndexOpDispenser extends MilvusBaseOpDispenser bF = l -> CreateIndexParam.newBuilder().withIndexName(targetF.apply(l)); - bF = op.enhanceFunc(bF, List.of("collection", "collection_name"), String.class, + bF = op.enhanceFunc(bF, "collection", String.class, CreateIndexParam.Builder::withCollectionName); bF = op.enhanceFunc(bF, "field_name", String.class, CreateIndexParam.Builder::withFieldName); bF = op.enhanceEnumOptionally(bF, "index_type", IndexType.class, CreateIndexParam.Builder::withIndexType); @@ -64,7 +67,7 @@ public class MilvusCreateIndexOpDispenser extends MilvusBaseOpDispenser b.withSyncWaitingInterval(n.longValue())); bF = op.enhanceFuncOptionally(bF, "sync_waiting_timeout", Number.class, (CreateIndexParam.Builder b, Number n) -> b.withSyncWaitingTimeout(n.longValue())); - bF = op.enhanceFuncOptionally(bF, List.of("database", "database_name"), String.class, + bF = op.enhanceFuncOptionally(bF, "database", String.class, CreateIndexParam.Builder::withDatabaseName); LongFunction finalBF1 = bF; return l -> finalBF1.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreatePartitionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreatePartitionOpDispenser.java index c20b07ebf..22b75fa1a 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreatePartitionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreatePartitionOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.CreatePartitionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusCreatePartitionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusCreatePartitionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -43,7 +46,7 @@ public class MilvusCreatePartitionOpDispenser extends MilvusBaseOpDispenser ebF = l -> CreatePartitionParam.newBuilder().withCollectionName(targetF.apply(l)); // Add enhancement functions here - ebF = op.enhanceFunc(ebF, List.of("collection","collection_name"),String.class, + ebF = op.enhanceFunc(ebF, "collection",String.class, CreatePartitionParam.Builder::withCollectionName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteCredentialOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteCredentialOpDispenser.java index fd846c43b..8b4fbb643 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteCredentialOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteCredentialOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.credential.DeleteCredentialParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDeleteCredentialOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusDeleteCredentialOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteOpDispenser.java index b3e74f124..ba4567063 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDeleteOpDispenser.java @@ -21,6 +21,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.dml.DeleteParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDeleteParamOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -32,15 +33,17 @@ import java.util.function.LongFunction; public class MilvusDeleteOpDispenser extends MilvusBaseOpDispenser { public MilvusDeleteOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override public LongFunction getParamFunc(LongFunction clientF, ParsedOp op, LongFunction targetF) { LongFunction f = l -> DeleteParam.newBuilder().withCollectionName(targetF.apply(l)); - f = op.enhanceFuncOptionally(f, List.of("partition_name","partition"), String.class, + f = op.enhanceFuncOptionally(f, "partition", String.class, DeleteParam.Builder::withPartitionName); f = op.enhanceFuncOptionally(f, "expression", String.class, DeleteParam.Builder::withExpr); f = op.enhanceFuncOptionally(f, "expr", String.class, Builder::withExpr); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeCollectionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeCollectionOpDispenser.java index 267c1d5a0..e61e55ca1 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeCollectionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeCollectionOpDispenser.java @@ -18,8 +18,8 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.DescribeCollectionParam; -import io.milvus.param.partition.CreatePartitionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDescribeCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +31,10 @@ public class MilvusDescribeCollectionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -44,7 +46,7 @@ public class MilvusDescribeCollectionOpDispenser extends MilvusBaseOpDispenser ebF = l -> DescribeCollectionParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("database","database_name"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "database",String.class, DescribeCollectionParam.Builder::withDatabaseName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeIndexOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeIndexOpDispenser.java index 97a25a024..bce52e6b2 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeIndexOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDescribeIndexOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.index.DescribeIndexParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDescribeIndexOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -35,8 +36,10 @@ public class MilvusDescribeIndexOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); op.getOptionalStaticValue("await_timeout", Number.class) .map(Number::doubleValue) @@ -53,9 +56,9 @@ public class MilvusDescribeIndexOpDispenser extends MilvusBaseOpDispenser ebF = l -> DescribeIndexParam.newBuilder().withIndexName(targetF.apply(l)); - ebF = op.enhanceFunc(ebF, List.of("collection","collection_name"), String.class, + ebF = op.enhanceFunc(ebF, "collection", String.class, DescribeIndexParam.Builder::withCollectionName); - ebF = op.enhanceFuncOptionally(ebF, List.of("database_name","database"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "database", String.class, DescribeIndexParam.Builder::withDatabaseName); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropAliasOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropAliasOpDispenser.java index 303b42e59..2fcc8699f 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropAliasOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropAliasOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.alias.DropAliasParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDropAliasOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusDropAliasOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropCollectionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropCollectionOpDispenser.java index fec247f08..f3fd94863 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropCollectionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropCollectionOpDispenser.java @@ -21,6 +21,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.DropCollectionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDropCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -34,18 +35,23 @@ public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenserCreate a new {@link MilvusDropCollectionOpDispenser} subclassed from {@link MilvusBaseOpDispenser}.

- * - *

{@see Drop Collection}

- * - * @param adapter The associated {@link MilvusDriverAdapter} - * @param op The {@link ParsedOp} encapsulating the activity for this cycle - * @param targetFunction A LongFunction that returns the specified Milvus Index for this Op +

Create a new {@link MilvusDropCollectionOpDispenser} subclassed from + {@link MilvusBaseOpDispenser}.

+ +

{@see Drop Collection}

+ @param adapter + The associated {@link MilvusDriverAdapter} + @param op + The {@link ParsedOp} encapsulating the activity for this cycle + @param targetFunction + A LongFunction that returns the specified Milvus Index for this Op */ public MilvusDropCollectionOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -55,7 +61,7 @@ public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenser targetF) { LongFunction f = l -> DropCollectionParam.newBuilder().withCollectionName(targetF.apply(l)); - f = op.enhanceFuncOptionally(f, List.of("database","database_name"),String.class, + f = op.enhanceFuncOptionally(f, "database",String.class, DropCollectionParam.Builder::withDatabaseName); LongFunction finalF = f; return l -> finalF.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropDatabaseOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropDatabaseOpDispenser.java index 31101259c..993af3b5f 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropDatabaseOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropDatabaseOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.DropDatabaseParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDropDatabaseOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusDropDatabaseOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropIndexOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropIndexOpDispenser.java index ac96796d4..d9d0d4c20 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropIndexOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropIndexOpDispenser.java @@ -21,39 +21,38 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.index.DropIndexParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDropIndexOp; import io.nosqlbench.adapters.api.templating.ParsedOp; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.List; import java.util.function.LongFunction; public class MilvusDropIndexOpDispenser extends MilvusBaseOpDispenser { /** - *

Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}.

- *

{@see Drop Index}

- * - * - * @param adapter - * The associated {@link MilvusDriverAdapter} - * @param op - * The {@link ParsedOp} encapsulating the activity for this cycle - * @param targetFunction - * A LongFunction that returns the specified Milvus Index for this Op +

Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}.

+

{@see Drop Index}

+ @param adapter + The associated {@link MilvusDriverAdapter} + @param op + The {@link ParsedOp} encapsulating the activity for this cycle + @param targetFunction + A LongFunction that returns the specified Milvus Index for this Op */ public MilvusDropIndexOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override public LongFunction getParamFunc(LongFunction clientF, ParsedOp op, LongFunction targetF) { LongFunction f = l -> DropIndexParam.newBuilder().withIndexName(targetF.apply(l)); - f = op.enhanceFunc(f, List.of("collection_name","collection"),String.class, + f = op.enhanceFunc(f, "collection",String.class, DropIndexParam.Builder::withCollectionName); LongFunction finalF = f; return l -> finalF.apply(1).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropPartitionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropPartitionOpDispenser.java index a4021f05d..d5d5e8f6d 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropPartitionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropPartitionOpDispenser.java @@ -17,9 +17,9 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; -import io.milvus.param.partition.CreatePartitionParam; import io.milvus.param.partition.DropPartitionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusDropPartitionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +31,10 @@ public class MilvusDropPartitionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -43,7 +45,7 @@ public class MilvusDropPartitionOpDispenser extends MilvusBaseOpDispenser ebF = l -> DropPartitionParam.newBuilder().withPartitionName(targetF.apply(l)); - ebF = op.enhanceFunc(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFunc(ebF, "collection",String.class, DropPartitionParam.Builder::withCollectionName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusFlushOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusFlushOpDispenser.java index 4f6e38fbe..93471589d 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusFlushOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusFlushOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.FlushParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusFlushOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +32,10 @@ public class MilvusFlushOpDispenser extends MilvusBaseOpDispenser { public MilvusFlushOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -51,7 +54,7 @@ public class MilvusFlushOpDispenser extends MilvusBaseOpDispenser { }; LongFunction finalEbF = ebF; ebF = l -> finalEbF.apply(l).withCollectionNames(cnames.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("database_name", "database"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "database", String.class, FlushParam.Builder::withDatabaseName); ebF = op.enhanceFuncOptionally(ebF, "sync_flush_waiting_interval", Number.class, (FlushParam.Builder b, Number n) -> b.withSyncFlushWaitingInterval(n.longValue())); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetBulkInsertStateOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetBulkInsertStateOpDispenser.java index 27d7ad9ce..b18ff0514 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetBulkInsertStateOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetBulkInsertStateOpDispenser.java @@ -19,9 +19,9 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.bulkinsert.GetBulkInsertStateParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetBulkInsertStateOp; -import io.nosqlbench.adapter.milvus.ops.MilvusGetCollectionStatisticsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; @@ -30,8 +30,10 @@ public class MilvusGetBulkInsertStateOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCollectionStatisticsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCollectionStatisticsOpDispenser.java index 26864609e..60dcdef69 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCollectionStatisticsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCollectionStatisticsOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.GetCollectionStatisticsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetCollectionStatisticsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusGetCollectionStatisticsOpDispenser extends MilvusBaseOpDispen public MilvusGetCollectionStatisticsOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -43,7 +46,7 @@ public class MilvusGetCollectionStatisticsOpDispenser extends MilvusBaseOpDispen LongFunction ebF = l -> GetCollectionStatisticsParam.newBuilder().withCollectionName(targetF.apply(l)); // Add enhancement functions here - ebF = op.enhanceFuncOptionally(ebF, List.of("database_name","database"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "database", String.class, GetCollectionStatisticsParam.Builder::withDatabaseName); ebF = op.enhanceFuncOptionally(ebF,"flush",Boolean.class,GetCollectionStatisticsParam.Builder::withFlush); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateOpDispenser.java index ce214fce4..deb015142 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetCompactionStateParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetCompactionStateOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusGetCompactionStateOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateWithPlansOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateWithPlansOpDispenser.java index 60f13a0c5..0c78450dc 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateWithPlansOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetCompactionStateWithPlansOpDispenser.java @@ -18,8 +18,8 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetCompactionPlansParam; -import io.milvus.param.control.GetCompactionPlansParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetCompactionStateWithPlansOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +30,10 @@ public class MilvusGetCompactionStateWithPlansOpDispenser extends MilvusBaseOpDi public MilvusGetCompactionStateWithPlansOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushAllStateOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushAllStateOpDispenser.java index 3fa532c22..117bda2e7 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushAllStateOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushAllStateOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetFlushAllStateParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetFlushAllStateOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusGetFlushAllStateOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushStateOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushStateOpDispenser.java index 5eedd57ef..e7e6e36dc 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushStateOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetFlushStateOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetFlushStateParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetFlushStateOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +32,9 @@ public class MilvusGetFlushStateOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -52,7 +54,7 @@ public class MilvusGetFlushStateOpDispenser extends MilvusBaseOpDispenser finalEbF = ebF; ebF = l -> finalEbF.apply(l).withSegmentIDs(idsF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection", "collection_name"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection", String.class, GetFlushStateParam.Builder::withCollectionName); ebF = op.enhanceFuncOptionally(ebF, "flush_ts", Number.class, (GetFlushStateParam.Builder b, Number n) -> b.withFlushTs(n.longValue())); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexBuildProgressOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexBuildProgressOpDispenser.java index eeef876d8..e650a0364 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexBuildProgressOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexBuildProgressOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.index.GetIndexBuildProgressParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetIndexBuildProgressOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusGetIndexBuildProgressOpDispenser extends MilvusBaseOpDispense public MilvusGetIndexBuildProgressOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -42,7 +45,7 @@ public class MilvusGetIndexBuildProgressOpDispenser extends MilvusBaseOpDispense ) { LongFunction ebF = l -> GetIndexBuildProgressParam.newBuilder().withIndexName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection",String.class, GetIndexBuildProgressParam.Builder::withCollectionName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexStateOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexStateOpDispenser.java index a78ade567..a7d124cad 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexStateOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetIndexStateOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.index.GetIndexStateParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetIndexStateOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusGetIndexStateOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -42,7 +45,7 @@ public class MilvusGetIndexStateOpDispenser extends MilvusBaseOpDispenser ebF = l -> GetIndexStateParam.newBuilder().withIndexName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection",String.class, GetIndexStateParam.Builder::withCollectionName); final LongFunction lastF = ebF; final LongFunction collectionParamF = l -> lastF.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadStateOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadStateOpDispenser.java index a9dc0421b..8f9a844ea 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadStateOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadStateOpDispenser.java @@ -21,6 +21,7 @@ import io.milvus.grpc.LoadState; import io.milvus.param.collection.GetLoadStateParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; import io.nosqlbench.adapter.milvus.MilvusAdapterUtils; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetLoadStateOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -40,15 +41,17 @@ public class MilvusGetLoadStateOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); op.getOptionalStaticValue("await_timeout", Number.class) .map(Number::doubleValue) .ifPresent(v->this.awaitTimeout=Duration.of((long)(v*1000),ChronoUnit.MILLIS)); op.getOptionalStaticValue("await_interval", Number.class) .map(Number::doubleValue).ifPresent(v->this.awaitInterval=Duration.of((long)(v*1000),ChronoUnit.MILLIS)); op.getOptionalStaticValue("await_state", String.class).ifPresent(s -> { - var spec = s.toLowerCase(); + String spec = s.toLowerCase(); for (LoadState value : LoadState.values()) { if (value.name().toLowerCase().equals(spec) || value.name().toLowerCase().equals("loadstate" + spec)) { this.awaitState = value; @@ -70,7 +73,7 @@ public class MilvusGetLoadStateOpDispenser extends MilvusBaseOpDispenser ebF = l -> GetLoadStateParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("database_name", "database"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "database", String.class, GetLoadStateParam.Builder::withDatabaseName); Optional> partitionsF = op.getAsOptionalFunction("partition_name", String.class); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadingProgressOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadingProgressOpDispenser.java index e53a7f5f3..ce8463901 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadingProgressOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadingProgressOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.GetLoadingProgressParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetLoadingProgressOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusGetLoadingProgressOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -42,7 +45,7 @@ public class MilvusGetLoadingProgressOpDispenser extends MilvusBaseOpDispenser ebF = l -> GetLoadingProgressParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("partition_names","partitions"), List.class, + ebF = op.enhanceFuncOptionally(ebF, "partitions", List.class, GetLoadingProgressParam.Builder::withPartitionNames); final LongFunction lastF = ebF; return l -> lastF.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetMetricsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetMetricsOpDispenser.java index a1127c22b..60610c769 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetMetricsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetMetricsOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetMetricsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetMetricsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusGetMetricsOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetOpDispenser.java index c4c8722bc..937c07879 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetOpDispenser.java @@ -21,6 +21,7 @@ import io.milvus.common.clientenum.ConsistencyLevelEnum; import io.milvus.param.highlevel.dml.GetIdsParam; import io.nosqlbench.adapter.milvus.MilvusAdapterUtils; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -46,8 +47,10 @@ public class MilvusGetOpDispenser extends MilvusBaseOpDispenser { public MilvusGetOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -75,7 +78,7 @@ public class MilvusGetOpDispenser extends MilvusBaseOpDispenser { LongFunction finalEbF2 = ebF; ebF = l -> finalEbF2.apply(l).withPrimaryIds(pidsF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name", "collection"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection", String.class, GetIdsParam.Builder::withCollectionName); ebF = op.enhanceEnumOptionally(ebF, "consistency_level", ConsistencyLevelEnum.class, GetIdsParam.Builder::withConsistencyLevel); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPartitionStatisticsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPartitionStatisticsOpDispenser.java index dd93f8a52..e95b64343 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPartitionStatisticsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPartitionStatisticsOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.GetPartitionStatisticsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetPartitionStatisticsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusGetPartitionStatisticsOpDispenser extends MilvusBaseOpDispens public MilvusGetPartitionStatisticsOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -42,7 +45,7 @@ public class MilvusGetPartitionStatisticsOpDispenser extends MilvusBaseOpDispens ) { LongFunction ebF = l -> GetPartitionStatisticsParam.newBuilder().withPartitionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection",String.class, GetPartitionStatisticsParam.Builder::withCollectionName); ebF = op.enhanceFuncOptionally(ebF, "flush",Boolean.class, GetPartitionStatisticsParam.Builder::withFlush); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPersistentSegmentInfoOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPersistentSegmentInfoOpDispenser.java index 5a7ef8c38..684c9543d 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPersistentSegmentInfoOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetPersistentSegmentInfoOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.GetPartitionStatisticsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetPartitionStatisticsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusGetPersistentSegmentInfoOpDispenser extends MilvusBaseOpDispe public MilvusGetPersistentSegmentInfoOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -44,7 +47,7 @@ public class MilvusGetPersistentSegmentInfoOpDispenser extends MilvusBaseOpDispe l -> GetPartitionStatisticsParam.newBuilder().withPartitionName(targetF.apply(l)); ebF = op.enhanceFuncOptionally(ebF,"flush",Boolean.class,GetPartitionStatisticsParam.Builder::withFlush); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection",String.class, GetPartitionStatisticsParam.Builder::withCollectionName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetQuerySegmentInfoOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetQuerySegmentInfoOpDispenser.java index b0b367446..b6d028758 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetQuerySegmentInfoOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetQuerySegmentInfoOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetQuerySegmentInfoParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetQuerySegmentInfoOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusGetQuerySegmentInfoOpDispenser extends MilvusBaseOpDispenser< public MilvusGetQuerySegmentInfoOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetReplicasOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetReplicasOpDispenser.java index 8fc1a2208..042798a3e 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetReplicasOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetReplicasOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.GetReplicasParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetReplicasOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusGetReplicasOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusHasPartitionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusHasPartitionOpDispenser.java index 14fa9323b..e1effa162 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusHasPartitionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusHasPartitionOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.HasPartitionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusHasPartitionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,9 @@ public class MilvusHasPartitionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -42,7 +44,7 @@ public class MilvusHasPartitionOpDispenser extends MilvusBaseOpDispenser ebF = l -> HasPartitionParam.newBuilder().withPartitionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF, "collection",String.class, HasPartitionParam.Builder::withCollectionName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertOpDispenser.java index c0f92eb7a..f9170b850 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertOpDispenser.java @@ -22,6 +22,7 @@ import com.alibaba.fastjson.JSONObject; import io.milvus.client.MilvusServiceClient; import io.milvus.param.dml.InsertParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusInsertOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,9 +30,7 @@ import io.nosqlbench.nb.api.errors.OpConfigError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.function.LongFunction; @@ -39,16 +38,20 @@ public class MilvusInsertOpDispenser extends MilvusBaseOpDispenser private static final Logger logger = LogManager.getLogger(MilvusInsertOpDispenser.class); /** - * Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}. - * - * @param adapter The associated {@link MilvusDriverAdapter} - * @param op The {@link ParsedOp} encapsulating the activity for this cycle - * @param targetFunction A LongFunction that returns the specified Milvus Index for this Op + Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}. + @param adapter + The associated {@link MilvusDriverAdapter} + @param op + The {@link ParsedOp} encapsulating the activity for this cycle + @param targetFunction + A LongFunction that returns the specified Milvus Index for this Op */ public MilvusInsertOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override @@ -57,11 +60,11 @@ public class MilvusInsertOpDispenser extends MilvusBaseOpDispenser l -> InsertParam.newBuilder().withCollectionName(targetF.apply(l)); f = op.enhanceFuncOptionally( - f, List.of("partition_name","partition"), String.class, + f, "partition", String.class, InsertParam.Builder::withPartitionName ); f = op.enhanceFuncOptionally( - f, List.of("database_name","database"), String.class, + f, "database", String.class, InsertParam.Builder::withDatabaseName ); @@ -76,13 +79,13 @@ public class MilvusInsertOpDispenser extends MilvusBaseOpDispenser } if (optionalRowsF.isPresent()) { - var rf = optionalRowsF.get(); + LongFunction> rf = optionalRowsF.get(); LongFunction finalF2 = f; f = l -> finalF2.apply(l).withRows(rf.apply(l)); } if (optionalFieldsF.isPresent()) { - var ff = optionalFieldsF.get(); + LongFunction> ff = optionalFieldsF.get(); LongFunction finalF3 = f; f = l -> finalF3.apply(l).withFields(ff.apply(l)); } diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertRowsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertRowsOpDispenser.java index bfc7cdd96..5ae95d7d7 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertRowsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusInsertRowsOpDispenser.java @@ -18,43 +18,38 @@ package io.nosqlbench.adapter.milvus.opdispensers; */ -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import io.milvus.client.MilvusServiceClient; -import io.milvus.param.dml.InsertParam; import io.milvus.param.highlevel.dml.InsertRowsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; -import io.nosqlbench.adapter.milvus.ops.MilvusInsertOp; import io.nosqlbench.adapter.milvus.ops.MilvusInsertRowsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; -import io.nosqlbench.nb.api.errors.OpConfigError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class MilvusInsertRowsOpDispenser extends MilvusBaseOpDispenser { private static final Logger logger = LogManager.getLogger(MilvusInsertRowsOpDispenser.class); /** - * Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}. - * - * @param adapter - * The associated {@link MilvusDriverAdapter} - * @param op - * The {@link ParsedOp} encapsulating the activity for this cycle - * @param targetFunction - * A LongFunction that returns the specified Milvus Index for this Op + Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}. + @param adapter + The associated {@link MilvusDriverAdapter} + @param op + The {@link ParsedOp} encapsulating the activity for this cycle + @param targetFunction + A LongFunction that returns the specified Milvus Index for this Op */ public MilvusInsertRowsOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction,spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListBulkInsertTasksOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListBulkInsertTasksOpDispenser.java index 237ddcabe..8069068f4 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListBulkInsertTasksOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListBulkInsertTasksOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.bulkinsert.ListBulkInsertTasksParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusListBulkInsertTasksOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusListBulkInsertTasksOpDispenser extends MilvusBaseOpDispenser< public MilvusListBulkInsertTasksOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCollectionsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCollectionsOpDispenser.java index 6d07a857c..97dd8342e 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCollectionsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCollectionsOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.highlevel.collection.ListCollectionsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusListCollectionsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusListCollectionsOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCredUsersOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCredUsersOpDispenser.java index 5ef5f7f7e..1faa6894a 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCredUsersOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListCredUsersOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.credential.ListCredUsersParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusListCredUsersOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusListCredUsersOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListDatabasesOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListDatabasesOpDispenser.java index 83eea541d..025e4e70f 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListDatabasesOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusListDatabasesOpDispenser.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusListDatabasesOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -28,8 +29,10 @@ public class MilvusListDatabasesOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadBalanceOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadBalanceOpDispenser.java index 209a6c185..bbbc1e8ba 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadBalanceOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadBalanceOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.LoadBalanceParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusLoadBalanceOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusLoadBalanceOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadCollectionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadCollectionOpDispenser.java index a66853eb9..be7390325 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadCollectionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadCollectionOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.LoadCollectionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusLoadCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -30,8 +31,10 @@ public class MilvusLoadCollectionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -43,7 +46,7 @@ public class MilvusLoadCollectionOpDispenser extends MilvusBaseOpDispenser ebF = l -> LoadCollectionParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("database_name", "database"), String.class, + ebF = op.enhanceFuncOptionally(ebF, "database", String.class, LoadCollectionParam.Builder::withDatabaseName); ebF = op.enhanceFuncOptionally(ebF, "refresh", Boolean.class, LoadCollectionParam.Builder::withRefresh); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadPartitionsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadPartitionsOpDispenser.java index 0f96c31ac..5430afb08 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadPartitionsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadPartitionsOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.LoadPartitionsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusLoadPartitionsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,16 +30,15 @@ import java.util.function.LongFunction; public class MilvusLoadPartitionsOpDispenser extends MilvusBaseOpDispenser { /** - * TODO: Refactor this class after API refinements for more type and target variation - * - * @param adapter - * @param op - * @param targetFunction + TODO: Refactor this class after API refinements for more type and target variation + @param adapter */ public MilvusLoadPartitionsOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -50,14 +50,14 @@ public class MilvusLoadPartitionsOpDispenser extends MilvusBaseOpDispenser ebF = l -> LoadPartitionsParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFunc(ebF, List.of("partition_names", "partitions"), List.class, + ebF = op.enhanceFunc(ebF, "partitions", List.class, LoadPartitionsParam.Builder::withPartitionNames); ebF = op.enhanceFuncOptionally( ebF, "resource_groups", List.class, LoadPartitionsParam.Builder::withResourceGroups ); ebF = op.enhanceFuncOptionally( - ebF, List.of("database_name", "database"), String.class, + ebF, "database", String.class, LoadPartitionsParam.Builder::withDatabaseName ); ebF = op.enhanceFuncOptionally(ebF, "refresh", Boolean.class, LoadPartitionsParam.Builder::withRefresh); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadingProgressOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadingProgressOpDispenser.java index 0a64fe857..4d7251478 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadingProgressOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadingProgressOpDispenser.java @@ -19,8 +19,10 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.GetLoadingProgressParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusGetLoadingProgressOp; +import io.nosqlbench.adapter.milvus.ops.MilvusSearchOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.List; @@ -30,8 +32,9 @@ public class MilvusLoadingProgressOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -42,7 +45,7 @@ public class MilvusLoadingProgressOpDispenser extends MilvusBaseOpDispenser ebF = l -> GetLoadingProgressParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF,List.of("partition_names","partitions"), List.class, + ebF = op.enhanceFuncOptionally(ebF,"partitions", List.class, GetLoadingProgressParam.Builder::withPartitionNames); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusManualCompactOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusManualCompactOpDispenser.java index a3f5e51b3..ac4d359e3 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusManualCompactOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusManualCompactOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.control.ManualCompactParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusManualCompactOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusManualCompactOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusQueryOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusQueryOpDispenser.java index 25ddaf123..a2d2ac42b 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusQueryOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusQueryOpDispenser.java @@ -20,6 +20,7 @@ import io.milvus.client.MilvusServiceClient; import io.milvus.common.clientenum.ConsistencyLevelEnum; import io.milvus.param.dml.QueryParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusQueryOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +32,10 @@ public class MilvusQueryOpDispenser extends MilvusBaseOpDispenser { public MilvusQueryOpDispenser(MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -44,7 +47,7 @@ public class MilvusQueryOpDispenser extends MilvusBaseOpDispenser { LongFunction ebF = l -> QueryParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("partition_names", "partitions"), List.class, + ebF = op.enhanceFuncOptionally(ebF, "partitions", List.class, QueryParam.Builder::withPartitionNames); ebF = op.enhanceEnumOptionally(ebF, "consistency_level", ConsistencyLevelEnum.class, QueryParam.Builder::withConsistencyLevel); ebF = op.enhanceFuncOptionally(ebF, "expr", String.class, QueryParam.Builder::withExpr); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleaseCollectionOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleaseCollectionOpDispenser.java index cdbf526fa..b4c6414e7 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleaseCollectionOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleaseCollectionOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.collection.ReleaseCollectionParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusReleaseCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusReleaseCollectionOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleasePartitionsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleasePartitionsOpDispenser.java index 740845ca8..30c87ce30 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleasePartitionsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleasePartitionsOpDispenser.java @@ -20,6 +20,7 @@ import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.ReleasePartitionsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; import io.nosqlbench.adapter.milvus.MilvusAdapterUtils; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusReleasePartitionsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +32,10 @@ public class MilvusReleasePartitionsOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -47,7 +50,7 @@ public class MilvusReleasePartitionsOpDispenser extends MilvusBaseOpDispenser finalEbF = ebF; ebF = l -> finalEbF.apply(l).withPartitionNames(partNamesF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF,List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF,"collection",String.class, ReleasePartitionsParam.Builder::withCollectionName); final LongFunction lastF = ebF; diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusSearchOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusSearchOpDispenser.java index 1362efb60..e0a26de35 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusSearchOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusSearchOpDispenser.java @@ -21,6 +21,7 @@ import io.milvus.common.clientenum.ConsistencyLevelEnum; import io.milvus.param.MetricType; import io.milvus.param.dml.SearchParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusSearchOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -32,8 +33,9 @@ public class MilvusSearchOpDispenser extends MilvusBaseOpDispenser public MilvusSearchOpDispenser( MilvusDriverAdapter adapter, ParsedOp op, - LongFunction targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -43,7 +45,7 @@ public class MilvusSearchOpDispenser extends MilvusBaseOpDispenser LongFunction ebF = l -> SearchParam.newBuilder().withCollectionName(targetF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF, List.of("partition_names", "partitions"), List.class, SearchParam.Builder::withPartitionNames); + ebF = op.enhanceFuncOptionally(ebF, "partitions", List.class, SearchParam.Builder::withPartitionNames); ebF = op.enhanceFuncOptionally(ebF, "out_fields", List.class, SearchParam.Builder::withOutFields); @@ -56,7 +58,7 @@ public class MilvusSearchOpDispenser extends MilvusBaseOpDispenser (SearchParam.Builder b, Number n) -> b.withRoundDecimal(n.intValue())); ebF = op.enhanceFuncOptionally(ebF, "ignore_growing", Boolean.class, SearchParam.Builder::withIgnoreGrowing); ebF = op.enhanceFuncOptionally(ebF, "params", String.class, SearchParam.Builder::withParams); - ebF = op.enhanceFunc(ebF, List.of("vector_field_name", "vector_field"), String.class, + ebF = op.enhanceFunc(ebF, "vector_field", String.class, SearchParam.Builder::withVectorFieldName); // TODO: sanity check List of Floats vs List of List of Floats at func construction time. ebF = op.enhanceFuncOptionally(ebF, "vectors", List.class, SearchParam.Builder::withVectors); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowCollectionsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowCollectionsOpDispenser.java index a4db3cd5c..bd77b4bba 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowCollectionsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowCollectionsOpDispenser.java @@ -21,6 +21,7 @@ import io.milvus.grpc.ShowType; import io.milvus.param.collection.ShowCollectionsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; import io.nosqlbench.adapter.milvus.MilvusAdapterUtils; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusShowCollectionsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -32,8 +33,10 @@ public class MilvusShowCollectionsOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -47,7 +50,7 @@ public class MilvusShowCollectionsOpDispenser extends MilvusBaseOpDispenser> collectionsF = l -> MilvusAdapterUtils.splitNames(targetF.apply(l)); LongFunction finalEbF = ebF; ebF = l -> finalEbF.apply(l).withCollectionNames(collectionsF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF,List.of("database_name","database"),String.class, + ebF = op.enhanceFuncOptionally(ebF,"database",String.class, ShowCollectionsParam.Builder::withDatabaseName); ebF = op.enhanceEnumOptionally(ebF,"show_type", ShowType.class,ShowCollectionsParam.Builder::withShowType); logger.warn(this.getClass().getSimpleName() + " is deprecated, use get_loading_progress instead"); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowPartitionsOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowPartitionsOpDispenser.java index e68df7caf..4f9154c1b 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowPartitionsOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowPartitionsOpDispenser.java @@ -20,6 +20,7 @@ import io.milvus.client.MilvusServiceClient; import io.milvus.param.partition.ShowPartitionsParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; import io.nosqlbench.adapter.milvus.MilvusAdapterUtils; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusShowPartitionsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -31,8 +32,10 @@ public class MilvusShowPartitionsOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override @@ -47,7 +50,7 @@ public class MilvusShowPartitionsOpDispenser extends MilvusBaseOpDispenser> partitionsF = l -> MilvusAdapterUtils.splitNames(targetF.apply(l)); LongFunction finalEbF = ebF; ebF = l -> finalEbF.apply(l).withPartitionNames(partitionsF.apply(l)); - ebF = op.enhanceFuncOptionally(ebF,List.of("collection_name","collection"),String.class, + ebF = op.enhanceFuncOptionally(ebF,"collection",String.class, ShowPartitionsParam.Builder::withCollectionName); final LongFunction lastF = ebF; final LongFunction collectionParamF = l -> lastF.apply(l).build(); diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusUpdateCredentialOpDispenser.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusUpdateCredentialOpDispenser.java index ad6d35107..f05bdbd35 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusUpdateCredentialOpDispenser.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusUpdateCredentialOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers; import io.milvus.client.MilvusServiceClient; import io.milvus.param.credential.UpdateCredentialParam; import io.nosqlbench.adapter.milvus.MilvusDriverAdapter; +import io.nosqlbench.adapter.milvus.MilvusSpace; import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp; import io.nosqlbench.adapter.milvus.ops.MilvusUpdateCredentialOp; import io.nosqlbench.adapters.api.templating.ParsedOp; @@ -29,8 +30,10 @@ public class MilvusUpdateCredentialOpDispenser extends MilvusBaseOpDispenser targetFunction) { - super(adapter, op, targetFunction); + LongFunction targetFunction, + LongFunction spaceF + ) { + super(adapter, op, targetFunction, spaceF); } @Override diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusBaseOp.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusBaseOp.java index 19c62bd1a..deabff0d7 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusBaseOp.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusBaseOp.java @@ -52,7 +52,7 @@ public abstract class MilvusBaseOp implements CycleOp { try { Object result = applyOp(value); if (result instanceof R r) { - var error = r.getException(); + Exception error = r.getException(); if (error!=null) { throw error; } diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusDescribeIndexOp.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusDescribeIndexOp.java index 35f2c54fd..2029b8fb3 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusDescribeIndexOp.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusDescribeIndexOp.java @@ -21,7 +21,7 @@ import io.milvus.grpc.DescribeIndexResponse; import io.milvus.grpc.IndexDescription; import io.milvus.param.R; import io.milvus.param.index.DescribeIndexParam; -import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.OpGenerator; import io.nosqlbench.adapters.api.scheduling.TimeoutPredicate; @@ -98,7 +98,7 @@ public class MilvusDescribeIndexOp extends MilvusBaseOp impl } @Override - public Op getNextOp() { + public CycleOp getNextOp() { return nextOp; } } diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusGetLoadStateOp.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusGetLoadStateOp.java index 93cc6cf2e..9ea1f9946 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusGetLoadStateOp.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusGetLoadStateOp.java @@ -22,7 +22,7 @@ import io.milvus.grpc.LoadState; import io.milvus.param.R; import io.milvus.param.collection.GetLoadStateParam; import io.nosqlbench.adapter.milvus.exceptions.MilvusAwaitStateIncompleteError; -import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.OpGenerator; import io.nosqlbench.adapters.api.scheduling.TimeoutPredicate; @@ -65,7 +65,7 @@ public class MilvusGetLoadStateOp extends MilvusBaseOp implem } @Override - public Op getNextOp() { + public CycleOp getNextOp() { return this.nextOp; } } diff --git a/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java b/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java index 59d3e4335..3255dbf94 100644 --- a/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java +++ b/nb-adapters/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpMapper.java @@ -43,7 +43,7 @@ public class MongoOpMapper implements OpMapper< } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { LongFunction ctxNamer = op.getAsFunctionOr("space", "default"); @@ -65,12 +65,12 @@ public class MongoOpMapper implements OpMapper< if (target.isPresent()) { TypeAndTarget targetData = target.get(); return switch (targetData.enumId) { - case command -> new MongoCommandOpDispenser(adapter, spaceInitF, op); + case command -> new MongoCommandOpDispenser(adapter, spaceF, op); }; } // For everything else use the command API else { - return new MongoCommandOpDispenser(adapter, spaceInitF, op); + return new MongoCommandOpDispenser(adapter, spaceF, op); } diff --git a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java index f7431011c..36e3c75af 100644 --- a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java +++ b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JOpMapper.java @@ -36,7 +36,7 @@ public class Neo4JOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { TypeAndTarget typeAndTarget = op.getTypeAndTarget(Neo4JOpType.class, String.class); LongFunction spaceFunc = adapter.getSpaceFunc(op); return switch (typeAndTarget.enumId) { diff --git a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncReadTxnOp.java b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncReadTxnOp.java index 7b0abf7c6..aa7dab910 100644 --- a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncReadTxnOp.java +++ b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncReadTxnOp.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.neo4j.ops; import org.neo4j.driver.Query; import org.neo4j.driver.Record; +import org.neo4j.driver.Result; import org.neo4j.driver.Session; import java.util.List; @@ -35,7 +36,7 @@ public class Neo4JSyncReadTxnOp extends Neo4JBaseOp{ public final Record[] apply(long value) { List recordList = session.executeRead( txn -> { - var result = txn.run(query); + Result result = txn.run(query); return result.list(); } ); diff --git a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncWriteTxnOp.java b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncWriteTxnOp.java index 93653b088..eb4353891 100644 --- a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncWriteTxnOp.java +++ b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/ops/Neo4JSyncWriteTxnOp.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.neo4j.ops; import org.neo4j.driver.Query; import org.neo4j.driver.Record; +import org.neo4j.driver.Result; import org.neo4j.driver.Session; import java.util.List; @@ -35,7 +36,7 @@ public class Neo4JSyncWriteTxnOp extends Neo4JBaseOp{ public final Record[] apply(long value) { List recordList = session.executeWrite( txn -> { - var result = txn.run(query); + Result result = txn.run(query); return result.list(); } ); diff --git a/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java b/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java index 20df9f8b7..87a69e64f 100644 --- a/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java +++ b/nb-adapters/adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarOpMapper.java @@ -42,7 +42,7 @@ public class PulsarOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { int spaceName = op.getStaticConfigOr("space", 0); // PulsarSpace pulsarSpace = spaceCache.get(spaceName); PulsarSpace pulsarSpace = adapter.getSpaceFunc(op).apply(spaceName); diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java index 53fd517a0..a0aa93fcf 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java @@ -48,11 +48,11 @@ public class QdrantOpMapper implements OpMapper,QdrantSpace> { * @param adapterC * @param op * The {@link ParsedOp} to be evaluated - * @param spaceInitF + * @param spaceF * @return The correct {@link QdrantBaseOpDispenser} subclass based on the op type */ @Override - public OpDispenser> apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser> apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { TypeAndTarget typeAndTarget = op.getTypeAndTarget( QdrantOpType.class, String.class, diff --git a/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java b/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java index 5a0d583a9..d12b9cfd5 100644 --- a/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java +++ b/nb-adapters/adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JOpMapper.java @@ -40,7 +40,7 @@ public class S4JOpMapper implements OpMapper { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { /* * If the user provides a body element, then they want to provide the JSON or diff --git a/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java b/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java index e8bf38624..cf67b8eba 100644 --- a/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java +++ b/nb-adapters/adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java @@ -95,7 +95,7 @@ public class StdoutDriverAdapter extends BaseDriverAdapter { } @Override - public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { return new StdoutOpDispenser(adapter,op,adapter.getSpaceFunc(op)); } diff --git a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java index 403a3a61f..19a825d28 100644 --- a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java +++ b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientOpMapper.java @@ -33,7 +33,7 @@ public class TcpClientOpMapper implements OpMapper apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { LongFunction spacefunc = op.getAsFunctionOr("space", "default"); LongFunction ctxfunc = adapter.getSpaceFunc(op); return new TcpClientOpDispenser(adapter,op,ctxfunc); diff --git a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java index 843623cf5..b65d26453 100644 --- a/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java +++ b/nb-adapters/adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerOpMapper.java @@ -33,7 +33,7 @@ public class TcpServerOpMapper implements OpMapper apply(NBComponent adapterC, ParsedOp op, LongFunction spaceInitF) { + public OpDispenser apply(NBComponent adapterC, ParsedOp op, LongFunction spaceF) { LongFunction spacefunc = op.getAsFunctionOr("space", "default"); LongFunction ctxfunc = adapter.getSpaceFunc(op); return new TcpServerOpDispenser(adapter,op,ctxfunc); diff --git a/nb-adapters/adapter-weaviate/pom.xml b/nb-adapters/adapter-weaviate/pom.xml index 13c99ddb5..d00104dcd 100644 --- a/nb-adapters/adapter-weaviate/pom.xml +++ b/nb-adapters/adapter-weaviate/pom.xml @@ -36,7 +36,7 @@ io.weaviate client - 4.8.2 + 4.9.0 diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateDriverAdapter.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateDriverAdapter.java index 1c8b25e5a..61defb3dc 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateDriverAdapter.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateDriverAdapter.java @@ -21,6 +21,7 @@ import static io.nosqlbench.adapter.weaviate.WeaviateAdapterUtils.WEAVIATE; import java.util.function.Function; import java.util.function.IntFunction; +import java.util.function.LongFunction; import io.nosqlbench.adapter.weaviate.ops.WeaviateBaseOp; import io.nosqlbench.adapters.api.activityimpl.OpMapper; @@ -33,26 +34,25 @@ import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.nb.api.labels.NBLabels; @Service(value = DriverAdapter.class, selector = WEAVIATE) -public class WeaviateDriverAdapter extends BaseDriverAdapter, WeaviateSpace> { +public class WeaviateDriverAdapter extends BaseDriverAdapter, WeaviateSpace> { - public WeaviateDriverAdapter(NBComponent parentComponent, NBLabels labels) { - super(parentComponent, labels); - } + public WeaviateDriverAdapter(NBComponent parentComponent, NBLabels labels) { + super(parentComponent, labels); + } - @Override - public OpMapper> getOpMapper() { - return new WeaviateOpMapper(this); - } + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(WeaviateSpace.getConfigModel()); + } - @Override - public IntFunction getSpaceInitializer(NBConfiguration cfg) { - return (int s) -> new WeaviateSpace(s, cfg); - } - - @Override - public NBConfigModel getConfigModel() { - return super.getConfigModel().add(WeaviateSpace.getConfigModel()); - } + @Override + public OpMapper, WeaviateSpace> getOpMapper() { + return new WeaviateOpMapper(this); + } + @Override + public LongFunction getSpaceInitializer(NBConfiguration cfg) { + return l -> new WeaviateSpace(this, l, cfg); + } } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateOpMapper.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateOpMapper.java index 853df78c3..711eabda5 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateOpMapper.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateOpMapper.java @@ -17,6 +17,7 @@ package io.nosqlbench.adapter.weaviate; import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.nb.api.components.core.NBComponent; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,7 +36,7 @@ import io.nosqlbench.engine.api.templating.TypeAndTarget; import java.util.function.IntFunction; import java.util.function.LongFunction; -public class WeaviateOpMapper implements OpMapper> { +public class WeaviateOpMapper implements OpMapper,WeaviateSpace> { private static final Logger logger = LogManager.getLogger(WeaviateOpMapper.class); private final WeaviateDriverAdapter adapter; @@ -48,31 +49,30 @@ public class WeaviateOpMapper implements OpMapper> { this.adapter = adapter; } - /** - * Given an instance of a {@link ParsedOp} returns the appropriate - * {@link WeaviateBaseOpDispenser} subclass - * - * @param op - * The {@link ParsedOp} to be evaluated - * @param spaceInitF - * @return The correct {@link WeaviateBaseOpDispenser} subclass based on the op - * type - */ - @Override - public OpDispenser> apply(ParsedOp op, LongFunction spaceInitF) { - TypeAndTarget typeAndTarget = op.getTypeAndTarget(WeaviateOpType.class, String.class, - "type", "target"); - logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); + @Override + public OpDispenser> apply( + NBComponent adapterC, + ParsedOp pop, + LongFunction spaceF + ) { + TypeAndTarget typeAndTarget = pop.getTypeAndTarget( + WeaviateOpType.class, + String.class, + "type", "target" + ); + LongFunction targetF = typeAndTarget.targetFunction; - return switch (typeAndTarget.enumId) { - case delete_collection -> new WeaviateDeleteCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_collection -> new WeaviateCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case get_collection_schema -> - new WeaviateGetCollectionSchemaOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_objects -> new WeaviateCreateObjectsOpDispenser(adapter, op, typeAndTarget.targetFunction); + logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + pop.getName() + "'"); + + return switch (typeAndTarget.enumId) { + case delete_collection -> new WeaviateDeleteCollectionOpDispenser(adapter, pop, spaceF, targetF); + case create_collection -> new WeaviateCreateCollectionOpDispenser(adapter, pop, spaceF, targetF); + case get_collection_schema -> + new WeaviateGetCollectionSchemaOpDispenser(adapter, pop, spaceF, targetF); + case create_objects -> new WeaviateCreateObjectsOpDispenser(adapter, pop, spaceF, targetF); // default -> throw new RuntimeException("Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + // "mapping parsed op " + op); - }; - } + }; + } } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateSpace.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateSpace.java index 22f5aad64..24aeb7728 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateSpace.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/WeaviateSpace.java @@ -46,96 +46,76 @@ import io.weaviate.client.v1.auth.exception.AuthException; * @see Weaviate Java * client */ -public class WeaviateSpace extends BaseSpace { +public class WeaviateSpace extends BaseSpace { private final static Logger logger = LogManager.getLogger(WeaviateSpace.class); private final NBConfiguration cfg; protected WeaviateClient client; /** - * Create a new WeaviateSpace Object which stores all stateful contextual - * information needed to interact with the Weaviate database instance. - * - * @param cfg The configuration ({@link NBConfiguration}) for this nb run - */ - public WeaviateSpace(int idx, NBConfiguration cfg) { - super(idx); + * Create a new WeaviateSpace Object which stores all stateful contextual + * information needed to interact with the Weaviate database instance. + * + * @param cfg The configuration ({@link NBConfiguration}) for this nb run + */ + public WeaviateSpace(WeaviateDriverAdapter adapter, long idx, NBConfiguration cfg) { + super(adapter, idx); this.cfg = cfg; } - public synchronized WeaviateClient getClient() throws AuthException { + public synchronized WeaviateClient getClient() { if (client == null) { - client = createClient(); + try { + client = createClient(); + } catch (Exception e) { + throw new RuntimeException(e); + } } return client; } - private WeaviateClient createClient() throws AuthException { + private WeaviateClient createClient() throws AuthException { String uri = cfg.get("uri"); - String scheme = cfg.getOptional("scheme").orElse("https"); + String scheme = cfg.getOptional("scheme").orElse("https"); - var requiredToken = cfg.getOptional("token_file") - .map(Paths::get) - .map( - tokenFilePath -> { - try { - return Files.readAllLines(tokenFilePath).getFirst(); - } catch (IOException e) { - String error = "Error while reading token from file:" + tokenFilePath; - logger.error(error, e); - throw new RuntimeException(e); - } - } - ).orElseGet( - () -> cfg.getOptional("token") - .orElseThrow(() -> new RuntimeException("You must provide either a token_file or a token to " + - "configure a Weaviate client")) - ); - if (requiredToken != null - && (cfg.getOptional("username").isPresent() || cfg.getOptional("password").isPresent())) { - throw new OpConfigError("Username/Password combo cannot be used together with token/tokenFile"); - } + var requiredToken = cfg.getOptional("token_file").map(Paths::get).map(tokenFilePath -> { + try { + return Files.readAllLines(tokenFilePath).getFirst(); + } catch (IOException e) { + String error = "Error while reading token from file:" + tokenFilePath; + logger.error(error, e); + throw new RuntimeException(e); + } + }).orElseGet(() -> cfg.getOptional("token").orElse(null) +// orElseThrow(() -> new RuntimeException("You must provide either a token_file or a token to " + "configure a Weaviate client")) + ); + if (requiredToken != null && (cfg.getOptional("username").isPresent() || cfg.getOptional("password").isPresent())) { + throw new OpConfigError("Username/Password combo cannot be used together with token/tokenFile"); + } - logger.info("{}: Creating new Weaviate Client with (masked) token [{}], uri/endpoint [{}]", - this.getName(), WeaviateAdapterUtils.maskDigits(requiredToken), uri); - Config config = new Config(scheme, uri); + Config config = new Config(scheme, uri); - if (cfg.getOptional("username").isPresent() && cfg.getOptional("password").isPresent()) { - return WeaviateAuthClient.clientPassword(config, cfg.getOptional("username").get(), - cfg.getOptional("password").get(), null); - } else { - return WeaviateAuthClient.apiKey(config, requiredToken); - } + String username = cfg.getOptional("username").orElse(null); + String password = cfg.getOptional("password").orElse(null); + + if (username != null & password != null) { + logger.info("{}: Creating new Weaviate Client with username [{}], and (masked) password [{}]", this.getName(), username, WeaviateAdapterUtils.maskDigits(password)); + return WeaviateAuthClient.clientPassword(config, username, password, null); + } else if (cfg.getOptional("token").isPresent()) { + logger.info("{}: Creating new Weaviate Client with (masked) token [{}], uri/endpoint [{}]", this.getName(), WeaviateAdapterUtils.maskDigits(requiredToken), uri); + return WeaviateAuthClient.apiKey(config, requiredToken); + } else { + logger.info("{}: Creating new Weaviate Client without credentials.", this.getName()); + return new WeaviateClient(config); + } } public static NBConfigModel getConfigModel() { - return ConfigModel.of(WeaviateSpace.class) - .add( - Param.optional("token_file", String.class, "the file to load the api token from") - ) - .add( - Param.defaultTo("token", "weaviate") - .setDescription("the Weaviate api token to use to connect to the database") - ) - .add( - Param.defaultTo("scheme", "http") - .setDescription("the scheme of the database. Defaults to http.")) - .add(Param.defaultTo("uri", "localhost:8080").setDescription( - "the URI endpoint in which the database is running. Do not provide any suffix like https:// here.") - ) - .add(Param.optional("username") - .setDescription("Username to be used for non-WCD clusters. Need Password config too.")) - .add(Param.optional("password") - .setDescription("Password to be used for non-WCD clusters. Need Username config too.")) - .add( - Param.defaultTo("timeout_ms", 3000) - .setDescription("sets the timeout in milliseconds for all requests. Defaults to 3000ms.") - ) - .asReadOnly(); + return ConfigModel.of(WeaviateSpace.class).add(Param.optional("token_file", String.class, "the file to load the api token from")).add(Param.optional("token", String.class).setDescription("the Weaviate api token to use to connect to " + "the " + "database")).add(Param.defaultTo("scheme", "http").setDescription("the scheme of the database. Defaults to http.")).add(Param.defaultTo("uri", "localhost:8080").setDescription("the URI endpoint in which the database is running. Do not provide any suffix like https:// here.")).add(Param.optional("username").setDescription("Username to be used for non-WCD clusters. Need Password config too.")).add(Param.optional("password").setDescription("Password to be used for non-WCD clusters. Need Username config too.")).add(Param.defaultTo("timeout_ms", 3000).setDescription("sets the timeout in milliseconds for all requests. Defaults to 3000ms.")).asReadOnly(); } @Override public void close() throws Exception { - client = null; + client = null; } } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateBaseOp.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateBaseOp.java index 44b4e1c42..115aa0c4c 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateBaseOp.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateBaseOp.java @@ -26,21 +26,21 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.weaviate.client.WeaviateClient; import io.weaviate.client.base.Result; -public abstract class WeaviateBaseOp implements CycleOp { +public abstract class WeaviateBaseOp implements CycleOp { protected final static Logger logger = LogManager.getLogger(WeaviateBaseOp.class); protected final WeaviateClient client; - protected final T request; + protected final REQUEST request; protected final LongFunction apiCall; - public WeaviateBaseOp(WeaviateClient client, T requestParam) { + public WeaviateBaseOp(WeaviateClient client, REQUEST requestParam) { this.client = client; this.request = requestParam; this.apiCall = this::applyOp; } - public WeaviateBaseOp(WeaviateClient client, T requestParam, LongFunction call) { + public WeaviateBaseOp(WeaviateClient client, REQUEST requestParam, LongFunction call) { this.client = client; this.request = requestParam; this.apiCall = call; @@ -48,11 +48,11 @@ public abstract class WeaviateBaseOp implements CycleOp { @SuppressWarnings("unchecked") @Override - public final Object apply(long value) { + public final RESULT apply(long value) { logger.trace("applying op: " + this); try { - Object result = applyOp(value); + RESULT result = applyOp(value); if (result instanceof Result) { // Result result = client.misc().readyChecker().run(); @@ -81,7 +81,7 @@ public abstract class WeaviateBaseOp implements CycleOp { } }; - public abstract Object applyOp(long value); + public abstract RESULT applyOp(long value); @Override public String toString() { diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateCollectionOp.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateCollectionOp.java index 274f8e582..775608bcd 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateCollectionOp.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateCollectionOp.java @@ -19,14 +19,14 @@ import io.weaviate.client.WeaviateClient; import io.weaviate.client.base.Result; import io.weaviate.client.v1.schema.model.WeaviateClass; -public class WeaviateCreateCollectionOp extends WeaviateBaseOp { +public class WeaviateCreateCollectionOp extends WeaviateBaseOp> { public WeaviateCreateCollectionOp(WeaviateClient client, WeaviateClass request) { super(client, request); } @Override - public Object applyOp(long value) { + public Result applyOp(long value) { Result createResponse = null; try { createResponse = client.schema().classCreator().withClass(request).run(); diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateObjectsOp.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateObjectsOp.java index 8e4da739c..67d92740c 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateObjectsOp.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateCreateObjectsOp.java @@ -24,7 +24,7 @@ import io.weaviate.client.v1.data.api.ObjectCreator; import io.weaviate.client.v1.data.model.WeaviateObject; import io.weaviate.client.v1.data.replication.model.ConsistencyLevel; -public class WeaviateCreateObjectsOp extends WeaviateBaseOp { +public class WeaviateCreateObjectsOp extends WeaviateBaseOp> { public WeaviateCreateObjectsOp(WeaviateClient client, ObjectCreator request) { super(client, request); @@ -32,7 +32,7 @@ public class WeaviateCreateObjectsOp extends WeaviateBaseOp { @SuppressWarnings("serial") @Override - public Object applyOp(long value) { + public Result applyOp(long value) { Result response = null; try { response = client.data().creator().withClassName("Glove25").withProperties(new HashMap() { diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateDeleteCollectionOp.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateDeleteCollectionOp.java index 7bb2cc1e4..a15fccc7c 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateDeleteCollectionOp.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateDeleteCollectionOp.java @@ -18,14 +18,14 @@ package io.nosqlbench.adapter.weaviate.ops; import io.weaviate.client.WeaviateClient; import io.weaviate.client.base.Result; -public class WeaviateDeleteCollectionOp extends WeaviateBaseOp { +public class WeaviateDeleteCollectionOp extends WeaviateBaseOp> { public WeaviateDeleteCollectionOp(WeaviateClient client, String request) { super(client, request); } @Override - public Object applyOp(long value) { + public Result applyOp(long value) { Result delColResponse = null; try { delColResponse = client.schema().classDeleter().withClassName(request).run(); diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateGetCollectionSchemaOp.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateGetCollectionSchemaOp.java index 7df05f2d8..290b194cd 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateGetCollectionSchemaOp.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/ops/WeaviateGetCollectionSchemaOp.java @@ -21,14 +21,14 @@ import io.weaviate.client.WeaviateClient; import io.weaviate.client.base.Result; import io.weaviate.client.v1.schema.model.Schema; -public class WeaviateGetCollectionSchemaOp extends WeaviateBaseOp { +public class WeaviateGetCollectionSchemaOp extends WeaviateBaseOp> { public WeaviateGetCollectionSchemaOp(WeaviateClient client, String request) { super(client, request); } @Override - public Object applyOp(long value) { + public Result applyOp(long value) { Result getColSchemaResponse = null; try { getColSchemaResponse = client.schema().getter().run(); diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateBaseOpDispenser.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateBaseOpDispenser.java index f8ab36741..8e3be25d3 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateBaseOpDispenser.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateBaseOpDispenser.java @@ -26,43 +26,37 @@ import io.nosqlbench.adapter.weaviate.ops.WeaviateBaseOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.nb.api.components.core.NBComponent; import io.weaviate.client.WeaviateClient; import io.weaviate.client.v1.auth.exception.AuthException; -public abstract class WeaviateBaseOpDispenser extends BaseOpDispenser, WeaviateSpace> { +public abstract class WeaviateBaseOpDispenser extends BaseOpDispenser, WeaviateSpace> { - protected final LongFunction weaviateSpaceFunction; - protected final LongFunction clientFunction; - private final LongFunction> opF; - private final LongFunction paramF; + protected final LongFunction clientFunction; + protected final LongFunction> opF; + private final LongFunction paramF; - protected WeaviateBaseOpDispenser(WeaviateDriverAdapter adapter, ParsedOp op, LongFunction targetF) { - super((DriverAdapter) adapter, op); - this.weaviateSpaceFunction = adapter.getSpaceFunc(op); - this.clientFunction = (long l) -> { - try { - return this.weaviateSpaceFunction.apply(l).getClient(); - } catch (AuthException e) { - throw new RuntimeException(e); - } + protected WeaviateBaseOpDispenser(NBComponent adapter, ParsedOp op, LongFunction spaceF, LongFunction targetF) { + super(adapter, op, spaceF); + this.clientFunction = (long l) -> { + return spaceF.apply(l).getClient(); }; - this.paramF = getParamFunc(this.clientFunction, op, targetF); - this.opF = createOpFunc(paramF, this.clientFunction, op, targetF); - } + this.paramF = getParamFunc(this.clientFunction, op, targetF); + this.opF = createOpFunc(paramF, this.clientFunction, op, targetF); + } - protected WeaviateDriverAdapter getDriverAdapter() { - return (WeaviateDriverAdapter) adapter; - } + public abstract LongFunction getParamFunc(LongFunction clientF, ParsedOp op, LongFunction targetF); - public abstract LongFunction getParamFunc(LongFunction clientF, ParsedOp op, - LongFunction targetF); + public abstract LongFunction> createOpFunc( + LongFunction paramF, + LongFunction clientF, + ParsedOp op, + LongFunction targetF + ); - public abstract LongFunction> createOpFunc(LongFunction paramF, - LongFunction clientF, ParsedOp op, LongFunction targetF); - - @Override - public WeaviateBaseOp getOp(long value) { - return opF.apply(value); - } + @Override + public WeaviateBaseOp getOp(long cycle) { + return opF.apply(cycle); + } } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateCollectionOpDispenser.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateCollectionOpDispenser.java index bdf474b0d..432353941 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateCollectionOpDispenser.java @@ -24,11 +24,13 @@ import java.util.Optional; import java.util.function.LongFunction; import io.nosqlbench.adapter.weaviate.WeaviateDriverAdapter; +import io.nosqlbench.adapter.weaviate.WeaviateSpace; import io.nosqlbench.adapter.weaviate.ops.WeaviateBaseOp; import io.nosqlbench.adapter.weaviate.ops.WeaviateCreateCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.nb.api.errors.OpConfigError; import io.weaviate.client.WeaviateClient; +import io.weaviate.client.base.Result; import io.weaviate.client.v1.misc.model.BQConfig; import io.weaviate.client.v1.misc.model.BQConfig.BQConfigBuilder; import io.weaviate.client.v1.misc.model.MultiTenancyConfig; @@ -55,12 +57,12 @@ import io.weaviate.client.v1.schema.model.WeaviateClass.WeaviateClassBuilder;; * "https://weaviate.io/developers/weaviate/manage-data/collections">Collection * docs */ -public class WeaviateCreateCollectionOpDispenser extends WeaviateBaseOpDispenser { +public class WeaviateCreateCollectionOpDispenser extends WeaviateBaseOpDispenser> { - public WeaviateCreateCollectionOpDispenser(WeaviateDriverAdapter adapter, ParsedOp op, - LongFunction targetF) { - super(adapter, op, targetF); - } + public WeaviateCreateCollectionOpDispenser( + WeaviateDriverAdapter adapter, ParsedOp op, LongFunction spaceF, LongFunction targetF) { + super(adapter, op, spaceF, targetF); + } @SuppressWarnings("rawtypes") @Override @@ -344,10 +346,13 @@ public class WeaviateCreateCollectionOpDispenser extends WeaviateBaseOpDispenser }; } - @Override - public LongFunction> createOpFunc(LongFunction paramF, - LongFunction clientF, ParsedOp op, LongFunction targetF) { - return l -> new WeaviateCreateCollectionOp(clientF.apply(l), paramF.apply(l)); - } + @Override + public LongFunction>> createOpFunc( + LongFunction paramF, + LongFunction clientF, + ParsedOp op, LongFunction targetF + ) { + return l -> new WeaviateCreateCollectionOp(clientF.apply(l), paramF.apply(l)); + } } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateObjectsOpDispenser.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateObjectsOpDispenser.java index cefafe291..176eea7de 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateObjectsOpDispenser.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateCreateObjectsOpDispenser.java @@ -18,15 +18,17 @@ package io.nosqlbench.adapter.weaviate.opsdispensers; import java.util.function.LongFunction; import io.nosqlbench.adapter.weaviate.WeaviateDriverAdapter; +import io.nosqlbench.adapter.weaviate.WeaviateSpace; import io.nosqlbench.adapter.weaviate.ops.WeaviateBaseOp; import io.nosqlbench.adapter.weaviate.ops.WeaviateCreateObjectsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.weaviate.client.WeaviateClient; +import io.weaviate.client.base.Result; import io.weaviate.client.v1.data.api.ObjectCreator; /** * Create objects. - * + * * @see Create * Objects. @@ -34,10 +36,11 @@ import io.weaviate.client.v1.data.api.ObjectCreator; * "https://weaviate.io/developers/weaviate/api/rest#tag/objects/post/objects">Create * Objects - REST API. */ -public class WeaviateCreateObjectsOpDispenser extends WeaviateBaseOpDispenser { +public class WeaviateCreateObjectsOpDispenser extends WeaviateBaseOpDispenser> { - public WeaviateCreateObjectsOpDispenser(WeaviateDriverAdapter adapter, ParsedOp op, LongFunction targetF) { - super(adapter, op, targetF); + public WeaviateCreateObjectsOpDispenser(WeaviateDriverAdapter adapter, ParsedOp op, + LongFunction spaceF, LongFunction targetF) { + super(adapter, op, spaceF, targetF); } public LongFunction getParamFunc(LongFunction clientF, ParsedOp op, @@ -64,7 +67,7 @@ public class WeaviateCreateObjectsOpDispenser extends WeaviateBaseOpDispenser> createOpFunc(LongFunction paramF, + public LongFunction>> createOpFunc(LongFunction paramF, LongFunction clientF, ParsedOp op, LongFunction targetF) { return l -> new WeaviateCreateObjectsOp(clientF.apply(l), paramF.apply(l)); } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateDeleteCollectionOpDispenser.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateDeleteCollectionOpDispenser.java index 2e32b196e..b74034041 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateDeleteCollectionOpDispenser.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateDeleteCollectionOpDispenser.java @@ -18,10 +18,13 @@ package io.nosqlbench.adapter.weaviate.opsdispensers; import java.util.function.LongFunction; import io.nosqlbench.adapter.weaviate.WeaviateDriverAdapter; +import io.nosqlbench.adapter.weaviate.WeaviateSpace; import io.nosqlbench.adapter.weaviate.ops.WeaviateBaseOp; import io.nosqlbench.adapter.weaviate.ops.WeaviateDeleteCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.nb.api.components.core.NBComponent; import io.weaviate.client.WeaviateClient; +import io.weaviate.client.base.Result; /** * Delete a Weaviate collection. @@ -33,28 +36,19 @@ import io.weaviate.client.WeaviateClient; * "https://weaviate.io/developers/weaviate/api/rest#tag/schema/delete/schema/{className}">Delete * Collection REST API. */ -public class WeaviateDeleteCollectionOpDispenser extends WeaviateBaseOpDispenser { +public class WeaviateDeleteCollectionOpDispenser extends WeaviateBaseOpDispenser> { - public WeaviateDeleteCollectionOpDispenser( - WeaviateDriverAdapter adapter, ParsedOp op, - LongFunction targetF) { - super(adapter, op, targetF); + public WeaviateDeleteCollectionOpDispenser(NBComponent adapter, ParsedOp op, LongFunction spaceF, LongFunction targetF) { + super(adapter, op, spaceF, targetF); } @Override - public LongFunction getParamFunc( - LongFunction clientF, ParsedOp op, - LongFunction targetF - ) { -// LongFunction ebF = l -> targetF.apply(l); -// -// final LongFunction lastF = ebF; -// return l -> lastF.apply(l); - return l -> new WeaviateDeleteCollectionOp(clientF.apply(l), targetF.apply(l)); + public LongFunction getParamFunc(LongFunction clientF, ParsedOp op, LongFunction targetF) { + return targetF; } @Override - public LongFunction> createOpFunc(LongFunction paramF, LongFunction clientF, ParsedOp op, LongFunction targetF) { + public LongFunction>> createOpFunc(LongFunction paramF, LongFunction clientF, ParsedOp op, LongFunction targetF) { return l -> new WeaviateDeleteCollectionOp(clientF.apply(l), paramF.apply(l)); } diff --git a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateGetCollectionSchemaOpDispenser.java b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateGetCollectionSchemaOpDispenser.java index 322877ab3..935f3b632 100644 --- a/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateGetCollectionSchemaOpDispenser.java +++ b/nb-adapters/adapter-weaviate/src/main/java/io/nosqlbench/adapter/weaviate/opsdispensers/WeaviateGetCollectionSchemaOpDispenser.java @@ -18,28 +18,27 @@ package io.nosqlbench.adapter.weaviate.opsdispensers; import java.util.function.LongFunction; import io.nosqlbench.adapter.weaviate.WeaviateDriverAdapter; +import io.nosqlbench.adapter.weaviate.WeaviateSpace; import io.nosqlbench.adapter.weaviate.ops.WeaviateBaseOp; import io.nosqlbench.adapter.weaviate.ops.WeaviateGetCollectionSchemaOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.weaviate.client.WeaviateClient; +import io.weaviate.client.base.Result; -public class WeaviateGetCollectionSchemaOpDispenser extends WeaviateBaseOpDispenser { +public class WeaviateGetCollectionSchemaOpDispenser extends WeaviateBaseOpDispenser> { - public WeaviateGetCollectionSchemaOpDispenser(WeaviateDriverAdapter adapter, ParsedOp op, - LongFunction targetF) { - super(adapter, op, targetF); - } + public WeaviateGetCollectionSchemaOpDispenser(WeaviateDriverAdapter adapter, ParsedOp op, LongFunction spaceF, LongFunction targetF) { + super(adapter, op, spaceF, targetF); + } - @Override - public LongFunction getParamFunc(LongFunction clientF, ParsedOp op, - LongFunction targetF) { - return l -> targetF.apply(l); - } + @Override + public LongFunction getParamFunc(LongFunction clientF, ParsedOp op, LongFunction targetF) { + return l -> targetF.apply(l); + } - @Override - public LongFunction> createOpFunc(LongFunction paramF, - LongFunction clientF, ParsedOp op, LongFunction targetF) { - return l -> new WeaviateGetCollectionSchemaOp(clientF.apply(l), paramF.apply(l)); - } + @Override + public LongFunction>> createOpFunc(LongFunction paramF, LongFunction clientF, ParsedOp op, LongFunction targetF) { + return l -> new WeaviateGetCollectionSchemaOp(clientF.apply(l), paramF.apply(l)); + } } diff --git a/nb-adapters/adapter-weaviate/src/main/resources/activities/weaviate_vector_live.yaml b/nb-adapters/adapter-weaviate/src/main/resources/activities/weaviate_vector_live.yaml index fc7e61963..214982ddb 100644 --- a/nb-adapters/adapter-weaviate/src/main/resources/activities/weaviate_vector_live.yaml +++ b/nb-adapters/adapter-weaviate/src/main/resources/activities/weaviate_vector_live.yaml @@ -11,39 +11,48 @@ description: | In all of these phases, it is important to instance the metrics with distinct names. Also, aggregates of recall should include total aggregate as well as a moving average. + Parameters: + TEMPLATE(weaviatehost,localhost:8080) + scenarios: weaviate_vectors: delete_collection: >- run tags==block:delete_collection errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(weaviatehost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(weaviatehost) +# token_file=TEMPLATE(token_file) schema_collection: >- run tags==block:schema_collection errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(weaviatehost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(weaviatehost) +# token_file=TEMPLATE(token_file) get_collection_schema: >- run tags==block:get_collection_schema errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(weaviatehost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(weaviatehost) +# token_file=TEMPLATE(token_file) # TODO - ,retry should be added to errors for rampup rampup: >- run tags==block:rampup errors===warn,counter cycles===TEMPLATE(train_cycles,TEMPLATE(trainsize,1000)) threads===TEMPLATE(train_threads,AUTO) - uri=TEMPLATE(weaviatehost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(weaviatehost) +# token_file=TEMPLATE(token_file) count_vectors: >- run tags==block:count_vectors errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(weaviatehost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(weaviatehost) +# token_file=TEMPLATE(token_file) search_points: >- run tags==block:search_points errors===warn,counter cycles===TEMPLATE(testann_cycles,TEMPLATE(testsize,1000)) threads===TEMPLATE(testann_threads,AUTO) - uri=TEMPLATE(weaviatehost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(weaviatehost) +# token_file=TEMPLATE(token_file) params: driver: weaviate @@ -158,7 +167,7 @@ blocks: factor: 3 multiTenancyConfig: enabled: true - + get_collection_schema: ops: get_collection_schema_op: diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index c0ebe8af8..68b3b5f82 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -270,7 +270,7 @@ adapter-weaviate-include - false + true diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index 5ed3a2d05..16dad1e11 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -30,6 +30,14 @@ pom ${project.artifactId} https://nosqlbench.io/ + + + io.nosqlbench + adapters-api + 5.21.5-SNAPSHOT + compile + + ${revision} @@ -217,7 +225,7 @@ adapter-weaviate-module - false + true adapter-weaviate diff --git a/nb-apis/adapters-api/pom.xml b/nb-apis/adapters-api/pom.xml index fadd66ade..96150ac23 100644 --- a/nb-apis/adapters-api/pom.xml +++ b/nb-apis/adapters-api/pom.xml @@ -91,7 +91,15 @@ src/main/resources true + + src/main/java + + **/*.md + + + + diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java index 7a7e2b3ad..a4cb5ce8c 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapters.api.activityconfig; import com.amazonaws.util.StringInputStream; import com.google.gson.GsonBuilder; +import io.nosqlbench.adapters.api.activityconfig.yaml.*; import io.nosqlbench.nb.api.nbio.Content; import io.nosqlbench.nb.api.nbio.NBIO; import io.nosqlbench.nb.api.nbio.ResolverChain; @@ -25,8 +26,6 @@ import io.nosqlbench.nb.api.advisor.NBAdvisorException; import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList; import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader; -import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat; -import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.templating.StrInterpolator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,6 +45,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +/// This class is responsible for loading op templates. public class OpsLoader { private final static Logger logger = LogManager.getLogger(OpsLoader.class); @@ -58,7 +58,7 @@ public class OpsLoader { } public static OpsDocList loadPath(String path, Map params, String... searchPaths) { - String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS; + String[] extensions = path.indexOf('.') > -1 ? new String[]{} : YAML_EXTENSIONS; ResolverChain chain = new ResolverChain(path); Content foundPath = NBIO.chain(chain.getChain()).searchPrefixes(searchPaths).pathname(chain.getPath()).extensionSet(extensions).first() .orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'")); @@ -66,10 +66,11 @@ public class OpsLoader { return loadString(foundPath.asString(), fmt, params, foundPath.getURI()); } - public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map params, URI srcuri) { + public static OpsDocList loadString( + final String sourceData, OpTemplateFormat fmt, Map params, URI srcuri) { logger.trace(() -> "Applying string transformer to data:" + sourceData); - if (srcuri!=null) { + if (srcuri != null) { logger.info("workload URI: '" + srcuri + "'"); } StrInterpolator transformer = new StrInterpolator(params); @@ -84,7 +85,6 @@ public class OpsLoader { // TODO: itemize inline to support ParamParser OpsDocList layered = new OpsDocList(rawOpsDocList); - transformer.checkpointAccesses().forEach((k, v) -> { layered.addTemplateVariable(k, v); params.remove(k); @@ -95,9 +95,9 @@ public class OpsLoader { private static String evaluateJsonnet(URI uri, Map params) { List injected = new LinkedList<>(List.of(Path.of(uri).toString())); - params.forEach((k,v) -> { + params.forEach((k, v) -> { if (v instanceof CharSequence cs) { - injected.addAll(List.of("--ext-str",k+"="+cs)); + injected.addAll(List.of("--ext-str", k + "=" + cs)); } }); @@ -113,15 +113,9 @@ public class OpsLoader { } int resultStatus = SjsonnetMain.main0( - injected.toArray(new String[0]), - new DefaultParseCache(), - inputStream, - stdoutStream, - stderrStream, - new os.Path(Path.of(System.getProperty("user.dir"))), - Option.empty(), - Option.empty(), - null + injected.toArray(new String[0]), new DefaultParseCache(), inputStream, stdoutStream, + stderrStream, new os.Path(Path.of(System.getProperty("user.dir"))), Option.empty(), + Option.empty(), null ); String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8); @@ -130,7 +124,7 @@ public class OpsLoader { logger.info("dryrun=jsonnet, dumping result to stdout and stderr:"); System.out.println(stdoutOutput); System.err.println(stderrOutput); - if (resultStatus==0 && stderrOutput.isEmpty()) { + if (resultStatus == 0 && stderrOutput.isEmpty()) { logger.info("no errors detected during jsonnet evaluation."); throw new NBAdvisorException("dryrun=jsonnet: No errors detected.", 0); } else { @@ -139,14 +133,16 @@ public class OpsLoader { } } if (!stderrOutput.isEmpty()) { - BasicError error = new BasicError("stderr output from jsonnet preprocessing: " + stderrOutput); - if (resultStatus!=0) { + BasicError error = new BasicError( + "stderr output from jsonnet preprocessing: " + stderrOutput); + if (resultStatus != 0) { throw error; } else { - logger.warn(error.toString(),error); + logger.warn(error.toString(), error); } } - logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines."); + logger.info("jsonnet processing read '" + uri + "', rendered " + stdoutOutput.split( + "\n").length + " lines."); logger.trace("jsonnet result:\n" + stdoutOutput); return stdoutOutput; diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpDef.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpDef.java index a8b7acb30..8c88e8be5 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpDef.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpDef.java @@ -24,19 +24,20 @@ import org.apache.logging.log4j.Logger; import java.util.*; /** - * See specification for what this should do in UniformWorkloadSpecificationTest - */ + See specification for what this should do in UniformWorkloadSpecificationTest */ public class RawOpDef extends RawOpFields { private final static Logger logger = LogManager.getLogger(RawOpDef.class); /** - * Contains all the op fields. If the key params is used, then fields are divided - * between the op fields map and the params map, with the non-specified one soaking up the dangling - * op fields. (Those not under 'op' or 'params' and which are not reserverd words) + Contains all the op fields. If the key params is used, then fields are divided + between the op fields map and the params map, with the non-specified one soaking up the dangling + op fields. (Those not under 'op' or 'params' and which are not reserverd words) */ private Object op; - private final static List opFieldSynonyms = List.of("stmt", "statement", "op", "operation"); + private final static List opFieldSynonyms = List.of( + "stmt", "statement", "op", "operation"); + private int refKey; public RawOpDef() { } @@ -71,14 +72,15 @@ public class RawOpDef extends RawOpFields { // if (!keyName.equals("stmt")) { // logger.info("Used implied stmt field under name '" + keyName + "'. You can just use 'stmt: ... "+ s +"' or the equivalent to avoid this warning."); // } - map.put("stmt",s.toString()); + map.put("stmt", s.toString()); // setOp(new LinkedHashMap(Map.of("stmt",s.toString()))); } else { setOp(op); } } if (found.size() > 1) { - throw new BasicError("You used " + found + " as an op name, but only one of these is allowed at a time."); + throw new BasicError( + "You used " + found + " as an op name, but only one of these is allowed at a time."); } else if ((getName() == null || getName().isEmpty()) && op == null && !map.isEmpty()) { Map.Entry first = map.entrySet().iterator().next(); setName(first.getKey()); @@ -91,7 +93,8 @@ public class RawOpDef extends RawOpFields { if (_op) { if (_params) { if (!map.isEmpty()) { - throw new OpConfigError("If you have scoped op and params, you may not have dangling fields. Op template named '" + this.getName() + "' is invalid. Move dangling params ("+ map.keySet() +") under another field."); + throw new OpConfigError( + "If you have scoped op and params, you may not have dangling fields. Op template named '" + this.getName() + "' is invalid. Move dangling params (" + map.keySet() + ") under another field."); } } else { // no params. Op was a scoped field and there are dangling fields, so assume they belong to params getParams().putAll(map); @@ -136,12 +139,11 @@ public class RawOpDef extends RawOpFields { private void checkForUnintendedJsonMap(Object m, List path) { if (m instanceof Map) { - ((Map)m).forEach((k,v) -> { + ((Map) m).forEach((k, v) -> { if (v == null) { - throw new OpConfigError("A map key '" + k.toString() + "' with a null value was encountered. This is not" + - " allowed, and may be the result of using an unquoted binding, like {" + k + "}. You can simply wrap this in quotes" + - " like \"{"+ k +"}\" to avoid interpreting this as a JSON map." + - (path.size()>0 ? String.join(".",path):"")); + throw new OpConfigError( + "A map key '" + k.toString() + "' with a null value was encountered. This is not" + " allowed, and may be the result of using an unquoted binding, like {" + k + "}. You can simply wrap this in quotes" + " like \"{" + k + "}\" to avoid interpreting this as a JSON map." + (path.size() > 0 ? String.join( + ".", path) : "")); } else { if (v instanceof Map) { path.add(k.toString()); @@ -152,4 +154,17 @@ public class RawOpDef extends RawOpFields { } } + public void setRefKey(int refKey) { + this.refKey = refKey; + } + + /** + * Get an integer key for the op template for this workload template, based on enumeration of all + * active op templates. This value is stable within the current instance of the workload template only. + * @return a unique integer key for this op template within the workload template + */ + public int getRefKey() { + return this.refKey; + } + } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java index b7fc11ab3..502e9091e 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java @@ -34,7 +34,7 @@ import java.util.function.Function; public class RawOpsLoader { private final static Logger logger = LogManager.getLogger(RawOpsLoader.class); - public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"}; + public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"}; private final ArrayList> transformers = new ArrayList<>(); @@ -113,6 +113,15 @@ public class RawOpsLoader { } } RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList); + + int refkey = 0; + for (RawOpsDoc rawOpsBlocks : rawOpsDocList) { + for (RawOpsBlock rawOpsBlock : rawOpsBlocks) { + for (RawOpDef rawOpDef : rawOpsBlock.getRawOpDefs()) { + rawOpDef.setRefKey(refkey++); + } + } + } return rawOpsDocList; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpData.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpData.java index 58c88b6fd..f11724e6c 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpData.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpData.java @@ -16,6 +16,8 @@ package io.nosqlbench.adapters.api.activityconfig.yaml; +import org.jetbrains.annotations.NotNull; + import java.security.InvalidParameterException; import java.util.LinkedHashMap; import java.util.Map; @@ -29,47 +31,61 @@ public class OpData extends OpTemplate { private Map params = Map.of(); private Map bindings = Map.of(); private Map tags = new LinkedHashMap<>(); - private int refKey; + private int refkey; - public OpData(String desc, String name, Map tags, Map bindings, Map params, Map op) { + public OpData( + String desc, String name, Map tags, Map bindings, + Map params, Map op, int refkey + ) { this.desc = desc; this.name = name; this.tags = tags; this.bindings = bindings; this.params = params; this.op = op; + this.refkey = refkey; } - public OpData() {} + public OpData() { + } - public OpData(Map opdata) { + public OpData(Map opdata) { applyFields(opdata); - if (opdata.size()>0) { - throw new RuntimeException("Unconsumed fields in construction of op data from map: " + opdata); + if (opdata.size() > 0) { + throw new RuntimeException( + "Unconsumed fields in construction of op data from map: " + opdata); } } - public OpData applyFields(Map opdata) { - LinkedHashMap toapply = new LinkedHashMap<>(opdata); - Optional.ofNullable(toapply.remove(OpTemplate.FIELD_DESC)).ifPresent(v -> this.setDesc(v.toString())); - Optional.ofNullable(toapply.remove(OpTemplate.FIELD_NAME)).ifPresent(v -> this.setName(v.toString())); + public OpData applyFields(Map opdata) { + LinkedHashMap toapply = new LinkedHashMap<>(opdata); + Optional.ofNullable(toapply.remove(OpTemplate.FIELD_DESC)).ifPresent( + v -> this.setDesc(v.toString())); + Optional.ofNullable(toapply.remove(OpTemplate.FIELD_NAME)).ifPresent( + v -> this.setName(v.toString())); Optional.ofNullable(toapply.remove(OpTemplate.FIELD_BINDINGS)).ifPresent(this::setBindings); Optional.ofNullable(toapply.remove(OpTemplate.FIELD_OP)).ifPresent(this::setOp); Optional.ofNullable(toapply.remove(OpTemplate.FIELD_PARAMS)).ifPresent(this::setParams); Optional.ofNullable(toapply.remove(OpTemplate.FIELD_TAGS)).ifPresent(this::setTags); + Optional.ofNullable(toapply.remove(OpTemplate.FIELD_REFKEY)).map( + v -> (Integer) v).ifPresent(this::setRefKey); - if (toapply.size()>0) { + if (toapply.size() > 0) { throw new InvalidParameterException("Fields were not applied to OpData:" + toapply); } return this; } + private void setRefKey(int refkey) { + this.refkey = refkey; + } + private void setTags(Object o) { if (o instanceof Map) { - ((Map) o).forEach((k,v) -> { - this.tags.put(k.toString(),v.toString()); + ((Map) o).forEach((k, v) -> { + this.tags.put(k.toString(), v.toString()); }); } else { throw new RuntimeException("Invalid type for tags: " + o.getClass().getSimpleName()); @@ -79,8 +95,8 @@ public class OpData extends OpTemplate { private void setParams(Object o) { if (o instanceof Map) { this.params = new LinkedHashMap<>(); - ((Map) o).forEach((k,v) -> { - this.params.put(k.toString(),v); + ((Map) o).forEach((k, v) -> { + this.params.put(k.toString(), v); }); } else { throw new RuntimeException("Invalid type for params: " + op.getClass().getSimpleName()); @@ -89,9 +105,9 @@ public class OpData extends OpTemplate { private void setOp(Object o) { if (o instanceof CharSequence) { - this.op = new LinkedHashMap<>(Map.of("stmt",o.toString())); + this.op = new LinkedHashMap<>(Map.of("stmt", o.toString())); } else if (o instanceof Map) { - this.op = new LinkedHashMap<>((Map)o); + this.op = new LinkedHashMap<>((Map) o); } else { throw new RuntimeException("Invalid type for op:" + op.getClass().getSimpleName()); } @@ -100,17 +116,18 @@ public class OpData extends OpTemplate { private void setBindings(Object bindings) { if (bindings instanceof Map) { this.bindings = new LinkedHashMap<>(); - ((Map) bindings).forEach((k,v) -> { - this.bindings.put(k.toString(),v.toString()); + ((Map) bindings).forEach((k, v) -> { + this.bindings.put(k.toString(), v.toString()); }); - } else if (bindings!=null) { - throw new RuntimeException("Invalid type for bindings: " + bindings.getClass().getSimpleName()); + } else if (bindings != null) { + throw new RuntimeException( + "Invalid type for bindings: " + bindings.getClass().getSimpleName()); } } private void setName(String name) { this.name = name; - this.tags.put("name",name); + this.tags.put("name", name); } private void setDesc(String desc) { @@ -142,16 +159,6 @@ public class OpData extends OpTemplate { return this.params; } - @Override - public int getRefKey() { - return refKey; - } - - @Override - public void setRefKey(int refKey) { - this.refKey = refKey; - } - @Override public Optional> getOp() { return Optional.of(this.op); @@ -161,4 +168,9 @@ public class OpData extends OpTemplate { public Optional getStmt() { return Optional.empty(); } + + @Override + public int getRefKey() { + return refkey; + } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java index 35b4fe4df..f4bb66236 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java @@ -32,7 +32,7 @@ public class OpDef extends OpTemplate { private final LinkedHashMap params; private final LinkedHashMap bindings; private final LinkedHashMap tags; - private int refKey; + private int refkey; public OpDef(OpsBlock block, RawOpDef rawOpDef) { this.block = block; @@ -40,6 +40,7 @@ public class OpDef extends OpTemplate { this.params = composeParams(); this.bindings = composeBindings(); this.tags = composeTags(); + this.refkey = rawOpDef.getRefKey(); } @Override @@ -69,6 +70,11 @@ public class OpDef extends OpTemplate { return Optional.of(newmap); } + @Override + public int getRefKey() { + return this.refkey; + } + @Override public LinkedHashMap getBindings() { return bindings; @@ -85,24 +91,12 @@ public class OpDef extends OpTemplate { return params; } - @Override - public int getRefKey() { - return this.refKey; - } - - @Override - public void setRefKey(int refKey) { - this.refKey = refKey; - } - private LinkedHashMap composeParams() { MultiMapLookup lookup = new MultiMapLookup<>(rawOpDef.getParams(), block.getParams()); LinkedHashMap params = new LinkedHashMap<>(lookup); return params; } - - @Override public Map getTags() { return tags; diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplate.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplate.java index 05214fc42..ae839cd49 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplate.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplate.java @@ -45,7 +45,7 @@ import java.util.function.Function; * for UniformWorkloadSpecification directly to see how this specification is tested and documented. *

*/ -public abstract class OpTemplate implements Tagged { +public abstract class OpTemplate implements Tagged, OpTemplateProps { private final static Gson gson = new GsonBuilder().setPrettyPrinting().create(); // TODO: coalesce Gson instances to a few statics on a central NB API class @@ -57,16 +57,7 @@ public abstract class OpTemplate implements Tagged { public final static String FIELD_PARAMS = "params"; public final static String FIELD_TAGS = "tags"; public final static String FIELD_REFKEY = "refkey"; - - /** - * @return a description for the op template, or an empty string - */ - public abstract String getDesc(); - - /** - * @return a name for the op template, user-specified or auto-generated - */ - public abstract String getName(); + private int refKey = -1; /** * Return a map of tags for this statement. Implementations are required to @@ -74,12 +65,9 @@ public abstract class OpTemplate implements Tagged { * * @return A map of assigned tags for the op, with the name added as an auto-tag. */ + @Override public abstract Map getTags(); - public abstract Map getBindings(); - - public abstract Map getParams(); - public Map getParamsAsValueType(Class type) { Map map = new LinkedHashMap<>(); for (String pname : getParams().keySet()) { @@ -130,13 +118,7 @@ public abstract class OpTemplate implements Tagged { } } - /** - * Get an integer key for the op template for this workload template, based on enumeration of all - * active op templates. This value is stable within the current instance of the workload template only. - * @return a unique integer key for this op template within the workload template - */ - public abstract int getRefKey(); - public abstract void setRefKey(int refKey); + public V getParam(String name, Class type) { Object object = getParams().get(name); @@ -196,11 +178,11 @@ public abstract class OpTemplate implements Tagged { return getStmt().map(s -> new ParsedTemplateString(s, getBindings())); } - public abstract Optional> getOp(); - public Map asData() { LinkedHashMap fields = new LinkedHashMap<>(); +// fields.put(FIELD_REFKEY, this.getRefKey()); + if (this.getDesc() != null && !this.getDesc().isBlank()) { fields.put(FIELD_DESC, this.getDesc()); } @@ -266,4 +248,5 @@ public abstract class OpTemplate implements Tagged { throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields()); } } + } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplateProps.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplateProps.java new file mode 100644 index 000000000..4790d298d --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpTemplateProps.java @@ -0,0 +1,46 @@ +package io.nosqlbench.adapters.api.activityconfig.yaml; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import java.util.Map; +import java.util.Optional; + +public interface OpTemplateProps { + /** + * @return a description for the op template, or an empty string + */ + String getDesc(); + + /** + * @return a name for the op template, user-specified or auto-generated + */ + String getName(); + + Map getTags(); + + Map getBindings(); + + Map getParams(); + + Optional> getOp(); + + Optional getStmt(); + + int getRefKey(); +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpsDocList.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpsDocList.java index 0a49ce055..4987b017d 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpsDocList.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpsDocList.java @@ -40,7 +40,7 @@ public class OpsDocList implements Iterable { public OpsDocList(RawOpsDocList rawOpsDocList) { this.rawOpsDocList = rawOpsDocList; - this.applyModifier(new enumerator()); +// this.applyModifier(new enumerator()); } public static OpsDocList none() { @@ -196,12 +196,4 @@ public class OpsDocList implements Iterable { return count; } - public static class enumerator implements Consumer { - private int count=0; - @Override - public void accept(OpTemplate opTemplate) { - opTemplate.setRefKey(count++); - } - } - } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java index 01a0fcdf8..4df163ae4 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpDispenser.java @@ -18,72 +18,93 @@ package io.nosqlbench.adapters.api.activityimpl; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.evalctx.CycleFunction; +import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate; +import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -/** - *

- *

Synopsis

- * An OpDispenser is responsible for mapping a cycle number into - * an executable operation. This is where Op Synthesis occurs - * in NoSQLBench.

- *
- *

- *

BaseOpDispenser

- *

- * Some common behaviors which are intended to be portable across all op - * dispenser types are implemented in {@link BaseOpDispenser}. It is - * strongly recommended that you use this as your base type when - * implementing op dispensers. - *

- *

Concepts

- * Op Synthesis is the process of building a specific executable - * operation for some (low level driver) API by combining the - * static and dynamic elements of the operation together. - * In most cases, implementations of OpDispenser will be constructed - * within the logic of an {@link OpMapper} which is responsible for - * determining the type of OpDispenser to use as associated with a specific - * type <T>. The OpMapper is called for each type of operation - * that is active during activity initialization. It's primary responsibility - * is figuring out what types of {@link OpDispenser}s to create based - * on the op templates provided by users. Once the activity is initialized, - * a set of op dispensers is held as live dispensers to use as needed - * to synthesize new operations from generated data in real time. - *

- * - *
- *

Implementation Strategy

- *

OpDispenser implementations are intended to be implemented - * for each type of distinct operation that is supported by a - * DriverAdapter. - * That is not to say that an OpDispenser can't be responsible for - * producing multiple types of operations. Operations which are similar - * in what they need and how they are constructed make sense to be implemented - * in the same op dispenser. Those which need different construction - * logic or fundamentally different types of field values should be implemented - * separately. The rule of thumb is to ensure that op construction patterns - * are easy to understand at the mapping level ({@link OpMapper}), - * and streamlined for fast execution at the synthesis level ({@link OpDispenser}). - * - * @param The parameter type of the actual operation which will be used - * to hold all the details for executing an operation, - * something that implements {@link io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp}. - */ -public interface OpDispenser> extends LongFunction, OpResultTracker { +/// ## Synopsis +/// +/// An OpDispenser is responsible for producing an executable operation +/// for a given cycle number. OpDispenser's are __NOT__ responsible for determine what kind of +/// operation the user intended. These two roles are separated by design so that when an +/// OpDispenser is called, all of the work of figuring out what specific type of operations to run +/// has already been done. [OpDispenser]s are produced by [OpMapper]s. Thus the role of an OpMapper +/// is to construct a `cycle->op` function. +/// +/// There is an important relationship between op templates which are interpreted by op mapping and +/// the executable operations which are produced by op dispensers. At a high level, the chain looks +/// like this: +/// ``` +/// op template -> op mapper -> op dispenser -> executable op +///``` +/// The types and roles associated with op synthesis are: +/// - [OpTemplate] - the raw op template data structure +/// - [ParsedOp] - an API around op template; a lambda construction kit +/// - [OpMapper] - interprets user intent by looking at the op template and constructs an +/// associated op dispenser +/// - [OpDispenser] - Applies a cycle number to a lambda to produce an executable op, and +/// keeps track of template-specific metrics. +/// - [CycleOp] - The base type of executable operations. +/// +/// ---- +/// +/// ## BaseOpDispenser +/// +/// Some common behaviors which are intended to be portable across all op +/// dispenser types are implemented in [BaseOpDispenser]. It is +/// __strongly__ recommended that you use this as your base type when +/// implementing op dispensers. (__refactoring will make this mandatory__) +/// +/// ## Concepts +/// +/// REWRITE BELOW +/// Op Synthesis is the process of building a specific executable +/// operation for some (low level driver) API by combining the +/// static and dynamic elements of the operation together. +/// In most cases, implementations of OpDispenser will be constructed +/// within the logic of an [OpMapper] which is responsible for +/// determining the type of OpDispenser to use as associated with a specific +/// type ``. The OpMapper is called for each op template +/// that is active (not excluded by tag filtering) during activity +/// initialization. It's primary responsibility is figuring out what types of +/// [OpDispenser]s to create based +/// on the op templates provided by users. Once the activity is initialized, +/// a set of op dispensers is held as live dispensers to use as needed +/// to synthesize new operations from generated data in real time. +/// +/// --- +/// +/// ## Implementation Strategy +/// +/// OpDispenser implementations are intended to be implemented +/// for each type of distinct operation that is supported by a +/// DriverAdapter. +/// That is not to say that an OpDispenser can't be responsible for +/// producing multiple types of operations. Operations which are similar +/// in what they need and how they are constructed make sense to be implemented +/// in the same op dispenser. Those which need different construction +/// logic or fundamentally different types of field values should be implemented +/// separately. The rule of thumb is to ensure that op construction patterns +/// are easy to understand at the mapping level ([OpMapper]) +/// and streamlined for fast execution at the synthesis level ([OpDispenser]) +/// @param +/// The parameter type of the actual operation which will be used to hold all the details for +/// executing an +/// operation, something that implements [CycleOp] - /** - * The apply method in an op dispenser should do all the work of - * creating an operation that is executable by some other caller. - * The value produced by the apply method should not require - * additional processing if a caller wants to execute the operation - * multiple times, as for retries. - * - * @param cycle The cycle number which serves as the seed for any - * generated op fields to be bound into an operation. - * @return an executable operation - */ +public interface OpDispenser> extends LongFunction, OpResultTracker { - OP getOp(long cycle); + /// This method should do all the work of + /// creating an operation that is executable by some other caller. + /// The value produced by the apply method should not require + /// additional processing if a caller wants to execute the operation + /// multiple times, as for retries. + /// @param cycle + /// The cycle number which serves as the seed for any + /// generated op fields to be bound into an operation. + /// @return an executable operation + OPTYPE getOp(long cycle); CycleFunction getVerifier(); diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java index d4ded90e8..515eca437 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java @@ -145,7 +145,7 @@ public interface OpMapper, SPACETYPE extends Space> * The {@link ParsedOp} which is the parsed version of the user-provided op template. This contains all the * fields provided by the user, as well as explicit knowledge of which ones are static and dynamic. It provides * convenient lambda-construction methods to streamline the effort of creating the top-level op lambda. - * @param spaceInitF + * @param spaceF * This is the pre-baked lambda needed to access the specific {@link SPACETYPE} for a given cycle, if or when it * is needed. Not all op types need this, since they may have all the state needed fully captured within the * native type. For those that do, ensure that you are accessing the value through this function lazily and @@ -154,9 +154,9 @@ public interface OpMapper, SPACETYPE extends Space> * @return An OpDispenser which can be used to synthesize directly executable operations. */ - OpDispenser apply( + OpDispenser apply( NBComponent adapterC, ParsedOp pop, - LongFunction spaceInitF + LongFunction spaceF ); } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java index 30df4d309..d000da8d5 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseSpace.java @@ -23,6 +23,9 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; /** * This example of a space uses the SelfT technique to enable * the self type to be used in method signatures and return types. + * + * {@inheritDoc} + * * @param */ public class BaseSpace > implements Space { diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java index 28bea1c4a..b694e4c73 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java @@ -35,120 +35,177 @@ import java.util.Optional; import java.util.function.Function; import java.util.function.LongFunction; -/** - *

The DriverAdapter interface is the top level API for implementing - * operations in NoSQLBench. It defines the related APIs needed to fully realized an adapter - * at runtime. A driver adapter can map op templates from YAML form to a fully executable - * form in native Java code, just as an application might do with a native driver. It can also - * do trivial operations, like simply calling {@code System.out.println(...)}. When you specify an adapter by name, - * you are choosing both the available operations, and the rules for converting a YAML op template into those - * operations. This is a two-step process: The adapter provides mapping logic for converting high-level templates from - * users into op types, and separate dispensing logic which can efficiently create these ops at runtime. When used - * together, they power op synthesis -- efficient and deterministic construction of runtime operations using - * procedural generation methods. - *

- * - *

- * Generally speaking, a driver adapter is responsible for - *

    - *
  • Defining a class type for holding related state known as a {@link Space}. - *
    • The type of this is specified as - * generic parameter {@link SPACETYPE}.
    • - *
  • Defining a factory method for constructing an instance of the space.
    • - *
  • Recognizing the op templates that are documented for it via an {@link OpMapper} - * and assigning them to an op implementation. - *
    • The base type of these ops is specified as generic - * parameter {@link OPTYPE}.
    - *
    • - *
  • Constructing dispensers for each matching op implementation with a matching {@link OpDispenser}
    • - * implementation. - *
- *

At runtime, the chain of these together ({@code cycle -> op mapping -> op dispensing -> op}) is cached - * as a look-up table of op dispensers. This results in the simpler logical form {@code cycle -> op synthesis -> - * operation}. - *

- * - *

Variable Naming Conventions

- *

- * Within the related {@link DriverAdapter} APIs, the following conventions are (more often) used, and will be found - * everywhere: - *

    - *
  • {@code namedF} describes a namedFunction variable. Functional patterns are used everywhere in these APIs - * .
    • - *
  • {@code namedC} describes a namedComponent variable. All key elements of the nosqlbench runtime are - * part of a component tree.
    • - *
  • {@code pop} describes a {@link ParsedOp} instance.
    • - *
- *

- *

Generic Parameters

- *

- * When a new driver adapter is defined with the generic parameters below, it becomes easy to build out a matching - * DriverAdapter with any modern IDE.

- * - * @param - * The type of {@link CycleOp} which will be used to wrap all operations for this driver adapter. This allows you - * to add context or features common to all operations of this type. This can be a simple Marker interface, or it can be something more - * concrete that captures common logic or state across all the operations used for a given adapter. It is highly - * advised to NOT leave it as simply {@code CycleOp}, since specific op implementations offer much - * better performance. - * @param - * The type of context space used by this driver to hold cached instances of clients, session, or other native - * driver state. This is the shared state which might be needed during construction operations for an adapter. - * No other mechanism is provided nor intended for holding adapter-specific state. You must store it in - * this type. This includes client instances, codec mappings, or anything else that a single instance of an - * application would need to effectively use a given native driver. - */ +import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate; + + +/// The DriverAdapter interface is the top level API for implementing operations of any kind in +/// NoSQLBench. It defines the related APIs needed to fully realize an adapter at runtime. A +/// driver adapter can map op templates from YAML form to a fully executable form in native Java +/// code, and then execute those native operations just as an application might do directly +/// with a native driver. It can also do trivial operations, like simply calling +/// `System.out.println(...)`. What a particular driver adapter does is open-ended, but this +/// usually means wrapping a native driver when supporting specific protocols. +/// +/// Every DriverAdapter has a simple name, as indicated on it's [Service] annotation. When you +/// specify an adapter by name, you are choosing both the available operations, and the rules +/// for converting a YAML op template into those operations. This is a two-step process: Mapping +/// user intentions, and generating executable operations, called op mapping and op dispensing, +/// respectively. Each adapter provides implementations for both of these phases for all the op +/// types is supports. When used together, they power _op synthesis_ -- efficient and +/// deterministic construction of runtime operations using procedural generation methods. +/// +/// An overview of all the key elements of the adapter API is given here. Understanding this +/// section means you know how the core op generator logic of NoSQLBench works. +/// +/// Generally speaking, a driver adapter is responsible for +/// - Implementing a type of adapter-specific [Space] to hold related state. +/// - The type of this is specified as generic parameter [SPACETYPE] on [DriverAdapter]. +/// - This is the primary state holder for any thing an application would typically need in order +/// to use a +/// (specific) native driver. Space instances are analogous to application instances, although +/// they only hold the essential state needed to enable native driver usage. By default, an +/// adapter only provides a single space, but users can override this to achieve higher native +/// driver concurrency for some specialized types of testing. +/// - In [DriverAdapter#getSpaceInitializer(NBConfiguration)], defining a factory method for +/// constructing an instance of the space when needed. +/// - The [NBConfiguration] is provided to configure app or driver settings as specified by +/// the user in activity parameters. +/// - in [OpMapper#apply(NBComponent, ParsedOp, LongFunction)], recognizing the op template +/// that is documented for it and constructing a matching [OpDispenser]`<`[OPTYPE]`>`. +/// - The [NBComponent] is part of the runtime component tree, and can be used to attach +/// user-visible component and naming structure as needed. The component tree supports +/// runtime event propogation, and automatic dimensional-labeling within the NoSQLBench +/// runtime. It also provides services for creating context-anchored metrics instruments, +/// configuration linting, and so on. +/// - The [ParsedOp] is a fully normalized view of an [OpTemplate], adhering to all the rules +/// of the _Uniform_ [workload_definition]. The parsed op API provides methods to assist in +/// constructing the lambdas which are the backbone of op synthesis. Lambdas constructed in +/// this way are highly efficient in modern JVM implementations, and serve effectively as +/// defered compilation in NoSQLBench. +/// - The [LongFunction]`<`[SPACETYPE]`>` provides functional access to the space needed for an +/// operation. It is a long function, since the space instance may be specific for each +/// (long) cycle value +/// - in [OpDispenser#apply(long)]), creating an executable operation in the form of a [CycleOp]. +/// - Op dispenser logic should call a previously constructed lambda, having been built either +/// in +/// the body of [OpMapper#apply(NBComponent, ParsedOp, LongFunction)] or in the constructor of +/// [OpDispenser]. In either case, the op synthesis function is realized before the op +/// dispenser is fully constructed, ensuring that op generation is streamlined. +/// - The long value provided here is the cycle value, and is the primary _coordinate_ provided +/// to the op synthesis lambda. Any properties or features of the operation should be fully +/// determined, and a matching immutable op implementation should be returned. +/// - implement [CycleOp#apply(long)] for each unique type of operation which can be dispensed. +/// - CycleOps should be immutable, since they may be retried when necessary. +/// Subsequent calls should exactly the same thing for a given op instance. +/// - the long cycle value is provided here for special cases like error handling, logging, +/// or debugging, but should not be needed for normal op execution. +/// +/// --- +/// +/// At a high level, the following sequence is executed for every cycle in an activity: +/// ``` +///[cycle value] -> op synthesis -> [executable op] -> op execution +///``` +/// or, as a functional sketch, `opExecution(opFunction(cycle))`. This is a simplified view of the +/// detailed steps, most of which are handled automatically by the nosqlbench runtime engine: +/// +/// ``` +/// cycle value +/// -> op template sequencing # memoized +/// -> op template selection # memoized +/// -> op template parsing # memoized +/// -> op template normalization # memoized +/// -> op type mapping # memoized +/// -> op dispensing +/// -> op execution +///``` +/// +/// Notice that some stages are optimized via a form of memoization, for efficient execution. +/// +/// --- +/// +/// ### Variable Naming Conventions +/// +/// Within the related [DriverAdapter] APIs, the following conventions are (more often) used, and will be +/// found everywhere: +/// - `namedF` describes a namedFunction variable. Functional patterns are used everywhere in these APIs. +/// - `namedC` describes a namedComponent variable. All key elements of the nosqlbench runtime are part of a +/// component tree. +/// - `pop` describes a [ParsedOp] instance. +/// +/// --- +/// +/// ### Generic Parameters +/// +/// When a new driver adapter is defined with the generic parameters below, it becomes easy to build out a matching +/// [DriverAdapter] with any modern IDE by using the _implement interface ..._ and similar features. +/// +/// @param OPTYPE +/// The type of [CycleOp] which will be used to wrap all operations for this driver adapter. +/// This allows you to add context or features common to all operations of this type. This can +/// be a simple +/// [marker](https://en.wikipedia.org/wiki/Marker_interface_pattern) interface, +/// or it can be something more concrete that captures common logic or state across all the +/// operations used for a given adapter. It is highly advised to _NOT_ leave it as simply +/// `CycleOp`, since specific op implementations offer much better performance. +/// @param SPACETYPE +/// The type of context space used by this driver to hold cached instances of clients, +/// session, or other native driver state. This is the shared state which might be needed +/// during construction operations for an adapter. No other mechanism is provided nor intended +/// for holding adapter-specific state. You must store it in this type. This includes client +/// instances, codec mappings, or anything else that a single instance of an application would +/// need to effectively use a given native driver. +/// public interface DriverAdapter, SPACETYPE extends Space> extends NBComponent { - /** - *

- *

Op Mapping

- * An Op Mapper is a function which can look at a {@link ParsedOp} and create a matching {@link OpDispenser}. - * An OpDispenser is a function that will produce a special type {@link OPTYPE} that this DriverAdapter implements - * as its op implementation. There may be many different ops supported by an adapter, thus there may be similarly - * many dispensers.

- * - *

- * Both {@link OpMapper} and {@link OpDispenser} are functions. The role of {@link OpMapper} is to - * map the op template provided by the user to an op implementation provided by the driver adapter, - * and then to create a factor function for it (the {@link OpDispenser}).

- * - *

These roles are split for a very good reason: Mapping what the user wants to do with an op template - * is resource intenstive, and should be as pre-baked as possible. This phase is the op mapping phase. - * It is essential that the mapping logic be very clear and maintainable. Performance is not as important - * at this phase, because all of the mapping logic is run during initialization of an activity. - *

- *

- * Conversely, op dispensing (the next phase) while an activity is running should be as efficient as - * possible. - *

- * - * @return a dispensing function for {@link OPTYPE} op generation - */ + /// ## Op Mapping + /// + /// An Op Mapper is a function which can look at a [ParsedOp] and create a matching + /// [OpDispenser]. + /// An OpDispenser is a function that will produce a special type [OPTYPE] that this + /// DriverAdapter implements. + /// There may be many different ops supported by an adapter, thus there may be similarly many + /// dispensers. + /// + /// Both [OpMapper] and [OpDispenser] are functions. The role of [OpMapper] is to map the op + /// template provided + /// by the user to an op implementation provided by the driver adapter, and then to create a + /// suitable function for + /// creating that type of operations, known as the [OpDispenser]. + /// + /// These roles are split for a very good reason: Mapping what the user wants to do with an op + /// template + /// is resource intensive, and should be as pre-baked as possible. This phase is the _op + /// mapping_ phase. + /// It is essential that the mapping logic be very clear and maintainable. Performance is not as + /// important + /// at this phase, because all of the mapping logic is run during initialization of an + /// activity. + /// + /// Conversely, _op dispensing_ (the next phase) while an activity is running should be as + /// efficient as possible. + /// @return a dispensing function for an [OPTYPE] op generation OpMapper getOpMapper(); - /** - * The preprocessor function allows the driver adapter to remap - * the fields in the op template before they are interpreted canonically. - * At this level, the transform is applied once to the input map - * (once per op template) to yield the map that is provided to - * {@link OpMapper} implementations. This is here to make backwards compatibility - * possible for op templates which have changed. Avoid using it unless necessary. - * - * @return A function to pre-process the op template fields. - */ + /// The preprocessor function allows the driver adapter to remap the fields in the op template + /// before they are + /// interpreted canonically. At this level, the transform is applied once to the input map (once + /// per op + /// template) to yield the map that is provided to [OpMapper] implementations. _This is here to + /// make + /// backwards compatibility possible for op templates which have changed. Avoid using it unless + /// necessary._ + /// @return A function to pre-process the op template fields. default Function, Map> getPreprocessor() { return f -> f; } - /** - * When a driver needs to identify an error uniquely for the purposes of - * routing it to the correct error handler, or naming it in logs, or naming - * metrics, override this method in your activity. - * - * @return A function that can reliably and safely map an instance of Throwable to a stable adapter-specific name. - */ + + /// When a driver needs to identify an error uniquely for the purposes of routing it to the + /// correct error + /// handler, or naming it in logs, or naming metrics, override this method in your activity. + /// @return A function that can reliably and safely map an instance of Throwable to a stable + /// adapter-specific name. default Function getErrorNameMapper() { return t -> t.getClass().getSimpleName(); } @@ -157,20 +214,22 @@ public interface DriverAdapter, SPACETYPE extends Spac return List.of(f -> f); } - /** - *

This method allows each driver adapter to create named state which is automatically - * cached and re-used by name. For each (driver,space) combination in an activity, - * a distinct space instance will be created. In general, adapter developers will - * use the space type associated with an adapter to wrap native driver instances - * one-to-one. As such, if the space implementation is a {@link AutoCloseable}, - * it will be explicitly shutdown as part of the activity shutdown.

- * - *

It is not necessary to implement a space for a stateless driver adapter, or one - * which puts all state into each op instance.

- * - * @return A function which can initialize a new Space, which is a place to hold - * object state related to retained objects for the lifetime of a native driver. - */ + /// This method allows each driver adapter to create named state which is automatically cached + /// and re-used by + /// name. For each (driver,space) combination in an activity, a distinct space instance will be + /// created. In + /// general, adapter developers will use the space type associated with an adapter to wrap + /// native driver + /// instances one-to-one. As such, if the space implementation is an [AutoCloseable], it will + /// be + /// explicitly shutdown as part of the activity shutdown. + /// + /// It is not necessary to implement a space for a stateless driver adapter, or one which + /// injects all necessary + /// state into each op instance. + /// @return A function which can initialize a new Space, which is a place to hold object state + /// related to + /// retained objects for the lifetime of a native driver. default LongFunction getSpaceInitializer(NBConfiguration cfg) { return n -> (SPACETYPE) new Space() { @Override @@ -180,71 +239,109 @@ public interface DriverAdapter, SPACETYPE extends Spac }; } + /// Provides the configuration for this driver adapter, which comes from the superset of + /// activity parameters given for the owning activity. Presently, the driver adapter acts + /// as a proxy to set these parameters on the space, but this will likely be updated. + /// Instead, the configuratin will be properly attached to the space directly, and the APIs + /// supporting it will enforce this. NBConfiguration getConfiguration(); - /** - * The standard way to provide docs for a driver adapter is to put them in one of two common places: - *
    - *
  • <resources>/<adaptername>.md - A single markdown file which is the named top-level - * markdown file for this driver adapter.
    • - *
  • <resources>/docs/<adaptername>/ - A directory containing any type of file which - * is to be included in docs under the adapter name, otherwise known as the {@link Service#selector()}
    • - *
  • <resources>/docs/<adaptername>.md
    • - *
- * - *

A build will fail if any driver adapter implementation is missing at least one self-named - * markdown doc file.

- * - * @return A {@link DocsBinder} which describes docs to include for a given adapter. - */ + /// The standard way to provide docs for a driver adapter is to put them in one of two common + /// places: + /// - `resources/.md` + /// - A single markdown file which is the named top-level markdown file for this driver + /// adapter. + /// - `resources/docs//` + /// - A directory containing any type of file which is to be included in docs under the + /// adapter name, otherwise + /// known as the [Service#selector()] + /// - `resources/docs/.md` + /// - An alternate location for the main doc file for an adapter, assuming you are using the + /// docs/ path. + /// + /// _A build will fail if any driver adapter implementation is missing at least one self-named + /// markdown doc file._ + /// @return A [DocsBinder] which describes docs to include for a given adapter. default DocsBinder getBundledDocs() { Docs docs = new Docs().namespace("drivers"); - String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName(); String cp_docspath = "docs/" + this.getAdapterName(); - Optional> bundled_docs = NBIO.local().pathname(dev_docspath, cp_docspath).first(); + Optional> bundled_docs = NBIO.local().pathname( + dev_docspath, cp_docspath).first(); bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf); - - Optional> maindoc = NBIO.local().pathname("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first(); - + Optional> maindoc = NBIO.local().pathname( + "/src/main/resources/" + this.getAdapterName() + ".md", + this.getAdapterName() + ".md" + ).first(); maindoc.map(Content::asPath).ifPresent(docs::addPath); - return docs.asDocsBinder(); } + /// Provide the simple name for this [DriverAdapter] implementation, derived from the + /// required [Service] annotation. default String getAdapterName() { Service svc = this.getClass().getAnnotation(Service.class); if (svc == null) { - throw new RuntimeException("The Service annotation for adapter of type " + this.getClass().getCanonicalName() + " is missing."); + throw new RuntimeException( + "The Service annotation for adapter of type " + this.getClass().getCanonicalName() + " is missing."); } return svc.selector(); } + /// Indicate the level of testing and muturity for the current adapter. This is not actively + /// used + /// and may be removed. + /// @deprecated default Maturity getAdapterMaturity() { return this.getClass().getAnnotation(Service.class).maturity(); } - /** - *

The cache of all objects needed within a single instance - * of a DriverAdapter which are not operations. These are generally - * things needed by operations, or things needed during the - * construction of operations.

- * - *

During Adapter Initialization, Op Mapping, Op Synthesis, or Op Execution, - * you may need access to the objects in (the or a) space cache. You can build the - * type of context needed and then provide this function to provide new instances - * when needed.

- * - *

The function returned by this method is specialized to the space mapping - * logic in the op template. Specifically, it uses whatever binding is set on a given - * op template for the space op field. If none are provided, then this - * becomes a short-circuit for the default '0'. If a non-numeric binding is provided, - * then an interstitial mapping is added which converts the {@link Object#toString()} - * value to ordinals using a hash map. This is less optimal by far than using - * any binding that produces a {@link Number}.

- * - * @return A cache of named objects - */ + /// The function returned by [#getSpaceFunc(ParsedOp)] provides access to a cache of all + /// stateful objects needed + /// within a single instance of a DriverAdapter. These are generally things needed by + /// operations, or things + /// needed during the construction of operations. Typically, a space is where you store a native + /// driver instance + /// which is expected to be created/initialized once and reused within an application. + /// Generally, users can + /// think of __space__ as __driver instance__, or __client instance__, although there are driver + /// adapters that + /// do things other than wrap native drivers and clients. + /// + /// The value of the op field `space` is used to customize the instancing behavior of spaces. If + /// none is provided + /// by the user, then only a singular space will be created for a given adapter in an activity. + /// This is normal, + /// and what most users will expect to do. However, customizing the space selector can be a + /// powerful way to test + /// any system with high logical concurrency. For example, each instance of a native driver will + /// typically + /// maintain its own thread or connection pools, cached resources and so on. ( Unless the + /// developers of + /// said native driver are breaking encapsulation by using global singletons in the runtime, + /// which is highly + /// frowned upon.) The spaces feature allows any nosqlbench workload to be easily converted into + /// an unreasonably parallel + /// client topology test with a single change. It works the same way for any adapter or protocol + /// supported by + /// nosqlbench. + /// + /// The value of the op field `space` should be a [Number] type. [Number#intValue()] is used to + /// determine which space instance is, if needed, initialized first, and then returned. If users provide a + /// non-[Number] type, + /// then an enumerating layer is added inline to convert any such value to an integer, which is + /// less optimal. + /// + /// During op mapping or dispensing, you may need access to state held by a driver-specific + /// implementation of + /// [SPACETYPE]. In the initialization phase, you only have access to the space function + /// itself. + /// This is important to maintain a boundary betwen the explicitly stateless and stateful parts + /// of the + /// runtime. To use the space, incorporate the space function into the lambdas which produce the + /// operation to be + /// executed. This is typically done in the construtor of the related [OpDispenser]. + /// @return A cache of named objects public LongFunction getSpaceFunc(ParsedOp pop); } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java index 37815fdcf..315f64689 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Space.java @@ -26,8 +26,9 @@ import io.nosqlbench.nb.api.components.core.NBNamedElement; * in testing scenarios. Within the operations for an adapter, the space * may be needed, for example, to construct prepared statements, or other * 'session-attached' objects. Put any state that you would normally - * associate with an instance of a native driver into a space, and use - * the {@link DriverAdapter#getSpaceCache()} to access it when needed.

+ * associate with an instance of a native driver into a space. + * A function to access the cycle-specific space instance is provided where + * you might needed, such as in the mapping or dispensing APIs. */ public interface Space extends NBNamedElement, AutoCloseable { diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Validator.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Validator.java new file mode 100644 index 000000000..6d11248d9 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/Validator.java @@ -0,0 +1,23 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public interface Validator { + public void validate(RESULT result); +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ValidatorSource.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ValidatorSource.java new file mode 100644 index 000000000..0e1fe6747 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/ValidatorSource.java @@ -0,0 +1,29 @@ +package io.nosqlbench.adapters.api.activityimpl.uniform; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.Optional; + +/// A [DriverAdapter] may implement this interface to provide adapter-specific +/// validators. +public interface ValidatorSource { + Optional getValidator(String name, ParsedOp pop); +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/CycleOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/CycleOp.java index 435d2b85c..07a019b38 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/CycleOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/CycleOp.java @@ -21,22 +21,52 @@ import io.nosqlbench.adapters.api.activityimpl.OpMapper; import java.util.function.LongFunction; -/** - *

CycleOp: f(cycle) -> T

- *

A CycleOp of T is an operation which takes a long input value and produces a value of type T. It is implemented as - * {@link LongFunction} of T.

- * - *

Designer Notes

- *

- * If you are using the value in this call to select a specific type of behavior, i.e. - * among a variety of operation types, it is very likely a candidate for factoring - * into separate op implementations. By using the 

cycle -> mapper -> dispenser -> op
- * pattern to move as much of the initialization logic forward, you will have - * much more efficient operations and much cleaner code. The {@link OpMapper} - * and {@link OpDispenser} abstractions are meant to move op type selection and scheduling to earlier in the activity. - *

- */ -public interface CycleOp extends LongFunction { +/// The [CycleOp] is the core interface for any executable operation within +/// the NoSQLBench runtime. When [#apply(long)] is called, the long value +/// is the value of a specific cycle. It is effectively `cycleop.apply(cycle) -> RESULT`. +/// +/// [CycleOp]s may or may not actually use the long input. Generally speaking, +/// once you have an instance of a cycle op, you have already synthesized all +/// concrete values needed to instantiate it. These values are derived from +/// the cycle within the op dispensing layer, thus the [CycleOp] should not need +/// them again to define the core operation. However, it is often helpful +/// to have the cycle value for debugging, diagnostics, or other instrumentation, +/// particularly in cases where special error handling is needed. In some rare +/// cases, special dispensers may use the value to reconstruct determinstic op +/// data for the purposes of troubleshooting or similar. +/// +/// A given [CycleOp] instance should execute the exact same operation if called again. +/// This is used to _retry_ operations in some cases. Some operations which may be +/// spawned as a result of previous operations may never use the cycle op value, as the +/// fields and behavior of those secondary operations may be fully determined by the +/// results or fields of the previous operation. This may be the case for linearized +/// operations which, for example, read further data from a data source as a client-side +/// join. +/// +/// ## Designer Notes +/// +/// If you are using the value in this call to select a specific type of behavior, i.e. among a variety of +/// operation types, it is very likely a candidate for factoring into separate op implementations. In general, +/// you should move as much of the initialization logic forward as possible, to keep op synthesis fast. +/// +/// If you derive from [CycleOp] to create subtypes within your own +/// [io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter] implementation, it is better to +/// keep the [RESULT] generic parameter within the arity of the derived type. For example, a +/// derived +/// ``` +/// public ... MyCycleOp +/// extends CycleOp +/// ``` +/// is much more understandable and reusable than +/// ``` +/// public ... MyCycleOp +/// extends CycleOp # confusing! +/// ``` +/// since the latter version replaces the generic result type with some other adapter-specific type. +/// At least in the first case, a clear specialization is carved out for a new generic parameter +/// without hiding the original result parameter. +/// +public interface CycleOp extends LongFunction { /** *

Run an action for the given cycle.

* @@ -44,7 +74,7 @@ public interface CycleOp extends LongFunction { * The cycle value for which an operation is run */ @Override - T apply(long value); + RESULT apply(long value); } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOp.java similarity index 66% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOp.java index 16511cedb..964dcbdb2 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOp.java @@ -16,19 +16,23 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; +import io.nosqlbench.adapters.api.activityimpl.uniform.Validator; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; -public class EmitterCycleOp implements CycleOp { +public class AssertingOp implements CycleOp { - private final CycleOp cycleOp; - public EmitterCycleOp(CycleOp cycleOp) { - this.cycleOp = cycleOp; + private final CycleOp op; + private final Validator validator; + + public AssertingOp(CycleOp op, Validator validator) { + this.op = op; + this.validator = validator; } @Override public T apply(long value) { - T result = cycleOp.apply(value); - System.out.println("result from cycle " + value + ":\n"+result); - return result; + T result = op.apply(value); + validator.validate(result); + return result; } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOpDispenser.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOpDispenser.java new file mode 100644 index 000000000..5f3dff72f --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/AssertingOpDispenser.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.Validator; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +public class AssertingOpDispenser extends BaseOpDispenser, S> { + + private final OpDispenser> realDispenser; + private final Validator validator; + + public AssertingOpDispenser( + DriverAdapter, S> adapter, + ParsedOp pop, + OpDispenser> realDispenser, + Validator validator + ) { + super(adapter, pop, adapter.getSpaceFunc(pop)); + this.realDispenser = realDispenser; + this.validator = validator; + logger.debug("initialized {} for result validation", pop.getName()); + } + + @Override + public CycleOp getOp(long cycle) { + CycleOp op = realDispenser.getOp(cycle); + return new AssertingOp<>(op, validator); + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOp.java new file mode 100644 index 000000000..7534b6fd5 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOp.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; + +import io.nosqlbench.adapters.api.activityimpl.uniform.Validator; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; + +import java.util.Map; +import java.util.function.Function; + +public class CapturingOp implements CycleOp> { + + private final CycleOp op; + private final Function> extractorF; + + public CapturingOp(CycleOp op, Function> extractorF) { + this.op = op; + this.extractorF = extractorF; + } + + @Override + public Map apply(long value) { + T result = op.apply(value); + Map map = extractorF.apply(result); + return map; + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOpDispenser.java similarity index 62% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOpDispenser.java index 7ba367c64..16d403fc6 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOpDispenserWrapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/CapturingOpDispenser.java @@ -23,18 +23,28 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.templating.ParsedOp; -public class EmitterOpDispenserWrapper extends BaseOpDispenser, Space> { +import java.util.Map; +import java.util.function.Function; - private final OpDispenser> realDispenser; +public class CapturingOpDispenser extends BaseOpDispenser>, S> { - public EmitterOpDispenserWrapper(DriverAdapter,Space> adapter, ParsedOp pop, - OpDispenser> realDispenser) { + private final OpDispenser> realDispenser; + private final Function> extractorF; + + public CapturingOpDispenser( + DriverAdapter, S> adapter, + ParsedOp pop, + OpDispenser> realDispenser, + Function> extractorF + ) { super(adapter, pop, adapter.getSpaceFunc(pop)); this.realDispenser = realDispenser; + this.extractorF = extractorF; } + @Override - public EmitterOp getOp(long cycle) { - CycleOp cycleOp = realDispenser.getOp(cycle); - return new EmitterOp(cycleOp); + public CycleOp> getOp(long cycle) { + CycleOp op = realDispenser.getOp(cycle); + return new CapturingOp<>(op,extractorF); } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryrunOp.java similarity index 90% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryrunOp.java index f8574b11c..ab0f2f5cf 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryrunOp.java @@ -18,11 +18,11 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; -public class DryCycleOp implements CycleOp { +public class DryrunOp implements CycleOp { private final CycleOp op; - public DryCycleOp(CycleOp op) { + public DryrunOp(CycleOp op) { this.op = op; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryrunOpDispenser.java similarity index 90% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryrunOpDispenser.java index 2dbb609de..5f4c9cce4 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryCycleOpDispenserWrapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/DryrunOpDispenser.java @@ -23,11 +23,11 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.templating.ParsedOp; -public class DryCycleOpDispenserWrapper extends BaseOpDispenser, S> { +public class DryrunOpDispenser extends BaseOpDispenser, S> { private final OpDispenser> realDispenser; - public DryCycleOpDispenserWrapper( + public DryrunOpDispenser( DriverAdapter, S> adapter, ParsedOp pop, OpDispenser> realDispenser @@ -45,6 +45,6 @@ public class DryCycleOpDispenserWrapper extends BaseOpD @Override public CycleOp getOp(long cycle) { CycleOp op = realDispenser.getOp(cycle); - return new DryCycleOp<>(op); + return new DryrunOp<>(op); } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/ResultPrintingOp.java similarity index 90% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/ResultPrintingOp.java index d009bfdf1..b836eb865 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/ResultPrintingOp.java @@ -18,10 +18,10 @@ package io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; -public class EmitterOp implements CycleOp { +public class ResultPrintingOp implements CycleOp { private final CycleOp cycleOp; - public EmitterOp(CycleOp cycleOp) { + public ResultPrintingOp(CycleOp cycleOp) { this.cycleOp = cycleOp; } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/ResultPrintingOpDispenser.java similarity index 86% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java rename to nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/ResultPrintingOpDispenser.java index f08f67990..954e57cf2 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/EmitterCycleOpDispenserWrapper.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/opwrappers/ResultPrintingOpDispenser.java @@ -23,11 +23,11 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; import io.nosqlbench.adapters.api.templating.ParsedOp; -public class EmitterCycleOpDispenserWrapper extends BaseOpDispenser, S> { +public class ResultPrintingOpDispenser extends BaseOpDispenser, S> { private final OpDispenser> realDispenser; - public EmitterCycleOpDispenserWrapper( + public ResultPrintingOpDispenser( DriverAdapter, S> adapter, ParsedOp pop, OpDispenser> realDispenser @@ -42,8 +42,8 @@ public class EmitterCycleOpDispenserWrapper extends BaseOpD } @Override - public EmitterCycleOp getOp(long cycle) { + public ResultPrintingOp getOp(long cycle) { CycleOp cycleOp = realDispenser.getOp(cycle); - return new EmitterCycleOp(cycleOp); + return new ResultPrintingOp(cycleOp); } } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/DiffType.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/DiffType.java new file mode 100644 index 000000000..cdfd33805 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/DiffType.java @@ -0,0 +1,53 @@ +package io.nosqlbench.adapters.api.evalctx.comparators; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public enum DiffType { + + /// Verify nothing for this result + none(0), + + /// Verify that keys named in the result are present in the reference map. + result_fields(0x1), + + /// Verify that keys in the reference map are present in the result map. + reference_fields(0x1 << 1), + + /// Verify that all fields present in either the reference map or the result map + /// are also present in the other. (set equality) + fields(0x1 | 0x1 << 1), + + /// Verify that all values of the same named key are equal, according to + /// {@link Object#equals(Object)}}. + values(0x1 << 2), + + /// Cross-verify all names and values between the reference map and result map. + all(0x1 | 0x1 << 1 | 0x1 << 2); + + public int bitmask; + + DiffType(int bit) { + this.bitmask = bit; + } + + public boolean is(DiffType option) { + return (bitmask & option.bitmask) > 0; + } + +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/MapDifferencer.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/MapDifferencer.java new file mode 100644 index 000000000..79c4f3808 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/evalctx/comparators/MapDifferencer.java @@ -0,0 +1,339 @@ +package io.nosqlbench.adapters.api.evalctx.comparators; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory; +import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter; +import io.nosqlbench.virtdata.core.bindings.Bindings; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.ToLongBiFunction; +import java.util.stream.Collectors; + +public class MapDifferencer { + + private final StringBuilder logbuffer = new StringBuilder(); + private final Map refMap = new HashMap<>(); + private final DiffType difftype; + private final Object metrics; + private final NBComponent scope; + private final NBMetricCounter verifiedResultsCounter; + private final NBMetricCounter verifiedValuesCounter; + private final NBMetricCounter unverifiedResultsCounter; + private final NBMetricCounter unverifiedValuesCounter; + + private MapDifferencer(NBComponent parent, Object metrics, Bindings bindings, DiffType diffType) { + this.scope = parent; + this.metrics = metrics; + this.difftype = diffType; + this.verifiedResultsCounter = parent.create().counter( + "verified_results", + MetricCategory.Verification, + "Number of results which were verified without error" + ); + this.verifiedValuesCounter = parent.create().counter( + "verified_values", + MetricCategory.Verification, + "Number of values in results which were verified" + ); + this.unverifiedResultsCounter = parent.create().counter( + "unverified_results", + MetricCategory.Verification, + "Number of results which were unverified (threw an error)" + ); + this.unverifiedValuesCounter = parent.create().counter( + "unverified_values", + MetricCategory.Verification, + "Number of values in results which were unverified (threw an error)" + ); + } + +// /** +// * see {@link DataType} +// * +// * @param typeName The DataType.Name of the field in question +// * @param row The row to read the field value from +// * @param fieldName The field name to read +// * @param genValue the generated value to compare against +// * @return true, if the value is equal +// */ +// private static boolean isEqual(DataType.Name typeName, Row row, String fieldName, Object genValue) { +// switch (typeName) { +// case ASCII: // ASCII(1, String.class) +// case VARCHAR: // VARCHAR(13, String.class) +// case TEXT: // TEXT(10, String.class) +// String textValue = row.getString(fieldName); +// return textValue.equals(genValue); +// case BIGINT: // BIGINT(2, Long.class) +// case COUNTER: // COUNTER(5, Long.class) +// long longValue = row.getLong(fieldName); +// return longValue == (long) genValue; +// case BLOB: // BLOB(3, ByteBuffer.class) +// // TODO: How do we test this one? +// case CUSTOM: // CUSTOM(0, ByteBuffer.class) +// ByteBuffer blobValue = row.getBytes(fieldName); +// return blobValue.equals(genValue); +// case BOOLEAN: // BOOLEAN(4, Boolean.class) +// boolean boolValue = row.getBool(fieldName); +// return boolValue == (boolean) genValue; +// case DECIMAL: // DECIMAL(6, BigDecimal.class) +// BigDecimal bigDecimalValue = row.getDecimal(fieldName); +// return bigDecimalValue.equals(genValue); +// case DOUBLE: // DOUBLE(7, Double.class) +// double doubleValue = row.getDouble(fieldName); +// return doubleValue == (double) genValue; +// case FLOAT: // FLOAT(8, Float.class) +// float floatValue = row.getFloat(fieldName); +// return floatValue == (float) genValue; +// case INET: // INET(16, InetAddress.class) +// InetAddress inetAddressValue = row.getInet(fieldName); +// return inetAddressValue.equals(genValue); +// case INT: // INT(9, Integer.class) +// int intValue = row.getInt(fieldName); +// return intValue == (int) genValue; +// case TIMESTAMP: // TIMESTAMP(11, Date.class) +// Date timestamp = row.getTimestamp(fieldName); +// return timestamp.equals(genValue); +// case UUID: // UUID(12, UUID.class) +// case TIMEUUID: // TIMEUUID(15, UUID.class) +// UUID uuidValue = row.getUUID(fieldName); +// return uuidValue.equals(genValue); +// case VARINT: // VARINT(14, BigInteger.class) +// BigInteger bigIntValue = row.getVarint(fieldName); +// return bigIntValue.equals(genValue); +// case LIST: // LIST(32, List.class) +// // TODO: How do we make getCollection methods work with non-String CQL types? +// List list = row.getList(fieldName, String.class); +// return list.equals(genValue); +// case SET: // SET(34, Set.class) +// Set set = row.getSet(fieldName, String.class); +// return set.equals(genValue); +// case MAP: // MAP(33, Map.class) +// Map map = row.getMap(fieldName, String.class, String.class); +// return map.equals(genValue); +// case UDT: // UDT(48, UDTValue.class) +// UDTValue udtValue = row.getUDTValue(fieldName); +// return udtValue.equals(genValue); +// case TUPLE: // TUPLE(49, TupleValue.class) +// TupleValue tupleValue = row.getTupleValue(fieldName); +// return tupleValue.equals(genValue); +// case SMALLINT: +// short shortVal = row.getShort(fieldName); +// return shortVal == (Short) genValue; +// case TINYINT: +// byte byteValue = row.getByte(fieldName); +// return byteValue == (byte) genValue; +// case DATE: +// LocalDate dateValue = row.getDate(fieldName); +// return dateValue.equals(genValue); +// case TIME: +// long timeValue = row.getTime(fieldName); +// return timeValue == (long) genValue; +// default: +// throw new RuntimeException("Unrecognized type:" + typeName); +// } +// } +// +// private static String prettyPrint(DataType.Name typeName, Row row, String fieldName) { +// switch (typeName) { +// case ASCII: // ASCII(1, String.class) +// case VARCHAR: // VARCHAR(13, String.class) +// case TEXT: // TEXT(10, String.class) +// return row.getString(fieldName); +// case BIGINT: // BIGINT(2, Long.class) +// case COUNTER: // COUNTER(5, Long.class) +// long counterValue = row.getLong(fieldName); +// return String.valueOf(counterValue); +// case BLOB: // BLOB(3, ByteBuffer.class) +// case CUSTOM: // CUSTOM(0, ByteBuffer.class) +// ByteBuffer blobValue = row.getBytes(fieldName); +// return String.valueOf(blobValue); +// case BOOLEAN: // BOOLEAN(4, Boolean.class) +// boolean boolValue = row.getBool(fieldName); +// return String.valueOf(boolValue); +// case DECIMAL: // DECIMAL(6, BigDecimal.class) +// BigDecimal bigDecimalValue = row.getDecimal(fieldName); +// return String.valueOf(bigDecimalValue); +// case DOUBLE: // DOUBLE(7, Double.class) +// double doubleValue = row.getDouble(fieldName); +// return String.valueOf(doubleValue); +// case FLOAT: // FLOAT(8, Float.class) +// float floatValue = row.getFloat(fieldName); +// return String.valueOf(floatValue); +// case INET: // INET(16, InetAddress.class) +// InetAddress inetAddressValue = row.getInet(fieldName); +// return String.valueOf(inetAddressValue); +// case INT: // INT(9, Integer.class) +// int intValue = row.getInt(fieldName); +// return String.valueOf(intValue); +// case TIMESTAMP: // TIMESTAMP(11, Date.class) +// Date timestamp = row.getTimestamp(fieldName); +// return String.valueOf(timestamp); +// case UUID: // UUID(12, UUID.class) +// case TIMEUUID: // TIMEUUID(15, UUID.class) +// UUID uuidValue = row.getUUID(fieldName); +// return String.valueOf(uuidValue); +// case VARINT: // VARINT(14, BigInteger.class) +// BigInteger bigIntValue = row.getVarint(fieldName); +// return String.valueOf(bigIntValue); +// case LIST: // LIST(32, List.class) +// List list = row.getList(fieldName, String.class); +// return String.valueOf(list); +// case SET: // SET(34, Set.class) +// Set set = row.getSet(fieldName, String.class); +// return String.valueOf(set); +// case MAP: // MAP(33, Map.class) +// Map map = row.getMap(fieldName, String.class, String.class); +// return String.valueOf(map); +// case UDT: // UDT(48, UDTValue.class) +// UDTValue udtValue = row.getUDTValue(fieldName); +// return String.valueOf(udtValue); +// case TUPLE: // TUPLE(49, TupleValue.class) +// TupleValue tupleValue = row.getTupleValue(fieldName); +// return String.valueOf(tupleValue); +// case SMALLINT: +// short val = row.getShort(fieldName); +// return String.valueOf(val); +// case TINYINT: +// byte byteValue = row.getByte(fieldName); +// return String.valueOf(byteValue); +// case DATE: +// LocalDate dateValue = row.getDate(fieldName); +// return String.valueOf(dateValue); +// case TIME: +// long timeValue = row.getTime(fieldName); +// return String.valueOf(timeValue); +// default: +// throw new RuntimeException("Type not recognized:" + typeName); +// } +// } + + /** + Compare the values of the result with the reference map. +

+ Specifically, +

    +
  1. Ensure the same number of fields.
    2. +
  2. Ensure the same pair-wise field names.
    3. +
  3. Ensure that each pair of same-named fields has the same data type.
    4. +
  4. Ensure that the value of each pair of fields is equal according to the equals + operator for the respective type.
    5. +
+ @return a count of differences between the row and the reference values + */ + private int compare(Map result, Map ref) { + int diff = 0; +// ColumnDefinitions cdefs = row.getColumnDefinitions(); + + logbuffer.setLength(0); + + if (difftype.is(DiffType.reference_fields)) { + List missingRowFields = ref.keySet().stream().filter(gk -> !result.containsKey(gk)).collect( + Collectors.toList()); + if (missingRowFields.size() > 0) { + diff += missingRowFields.size(); + + logbuffer.append("\nexpected fields '"); + logbuffer.append(String.join("','", missingRowFields)); + logbuffer.append("' not in row."); + } + } + + if (difftype.is(DiffType.result_fields)) { + List missingRefFields = result.keySet().stream().filter(k -> !ref.containsKey(k)).collect( + Collectors.toList()); + if (missingRefFields.size() > 0) { + diff += missingRefFields.size(); + + logbuffer.append("\nexpected fields '"); + logbuffer.append(String.join("','", missingRefFields)); + logbuffer.append("' not in reference data: " + ref); + } + } + + if (difftype.is(DiffType.values)) { + for (String name : result.keySet()) { + + if (ref.containsKey(name)) { + if (!result.get(name).equals(ref.get(name))) { + logbuffer.append("\nvalue differs for '").append(name).append("' "); + logbuffer.append("expected:'").append(ref.get(name).toString()).append("'"); + logbuffer.append(" actual:'").append(result.get(name).toString()).append("'"); + diff++; + this.unverifiedValuesCounter.inc(); + } else { + this.verifiedValuesCounter.inc(); + } + } + } + } + if (diff == 0) { + this.verifiedResultsCounter.inc(); + } else { + this.unverifiedResultsCounter.inc(); + } + return diff; + } + + /** + Get the most recent detail log recorded by this thread. + @return a logbuffer string, with one entry per line + */ + public String getDetail() { + return this.logbuffer.toString(); + } + + public int verify(long cycle, Object result) { +// refMap.clear(); +// bindings.setMap(refMap, cycle); +// int diffs = compare(row, refMap); +// if (diffs > 0) { +// HashMap mapcopy = new HashMap<>(); +// mapcopy.putAll(refMap); +// throw new RowVerificationException(cycle, row, mapcopy, getDetail()); +// } else { + return 0; +// } + } + + public static class ThreadLocalWrapper { + + private final NBComponent parent; + private final Bindings bindings; + private final DiffType diffType; + private final ThreadLocal tl; + + public ThreadLocalWrapper(NBComponent parent, Bindings bindings, DiffType diffType) { + this.parent = parent; + this.bindings = bindings; + this.diffType = diffType; + tl = ThreadLocal.withInitial(() -> new MapDifferencer(parent, bindings, bindings, diffType)); + } + + public int apply(Map result, long cycle) { + return tl.get().verify(cycle,result); + } + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/package-info.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/package-info.java new file mode 100644 index 000000000..c10f0e052 --- /dev/null +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/package-info.java @@ -0,0 +1,20 @@ +/// This package contains the API needed to implement a [io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter] +package io.nosqlbench.adapters.api; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java index 7f015e7f2..f013b6e09 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java @@ -35,6 +35,7 @@ import io.nosqlbench.nb.api.labels.NBLabelSpec; import io.nosqlbench.nb.api.labels.NBLabels; import io.nosqlbench.virtdata.core.templates.BindPoint; import io.nosqlbench.virtdata.core.templates.CapturePoint; +import io.nosqlbench.virtdata.core.templates.CapturePoints; import io.nosqlbench.virtdata.core.templates.ParsedTemplateString; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -43,295 +44,351 @@ import java.util.*; import java.util.function.*; /** - *

ParsedOp API

- * - *

This is the primary developer-focused API for driver developers to use when up-converting op templates - * to operations. This {@link ParsedOp} is a wrapper around the op template structure. It provides many - * ways of constructing higher-order objects from a variety of sources.

- * - *

Supporting Variety

- *

- * For some drivers or protocols, the primary user interface is a statement format or grammar. For CQL or SQL, - * the most natural way of describing templates for operations is in that native format. For others, an operation - * template may look like a block of JSON or a HTTP request. All these forms are supported. In order to deal with - * the variety, there is a set of detailed rules for how the workload definitions are transformed into driver - * operations for a native driver. The high-level flow is: - *

    - *
  1. Op Template Form
    2. - *
  2. Normalized Data Structure
    3. - *
  3. ParsedOp Form
    4. - *
- *

- * The specifications govern how the raw op template form is transformed into the normalized data structure. - * This documentation focuses on the API provided by the last form, this class, although peripheral awareness - * of the first two forms will certainly help for any advanced scenarios. You can find detailed examples and - * specification tests under the workload_definition folder of the adapters-api module. - *

- * - *
- *

Op Template Parsing

- *
    - *
  1. Rule #1: All op templates are parsed into an internal normalized structure which contains: - *
      - *
    1. A map of op fields, which can consist of: - *
        - *
      1. static field values
        2. - *
      2. dynamic field values, the actual value of which can only be known for a given cycle
        3. - *
      - *
      2. - *
    2. Access to auxiliary configuration values like activity parameters. These can back-fill - * when values aren't present in the direct static or dynamic op fields.
      3. - * - * - *
    3. Rule #2: When asking for a dynamic parameter, static parameters may be automatically promoted to functional form - * as a back-fill.
      4. - *
    4. Rule #3: When asking for static parameters, config parameters may automatically be promoted as a back-fill.
      5. - *
    - * The net effect of these rules is that the NoSQLBench driver developer may safely use functional forms to access data - * in the op template, or may decide that certain op fields must only be provided in a static way per operation. - *

    - *
    - *

    Distinguishing Op Payload from Op Config

    - *

    When a user specifies an op template, they may choose to provide only a single set of op fields without - * distinguishing between config or payload, or they may choose to directly configure each. - * - * 

    {@code
- * ops:
- *
- *  # both op and params explicitly named
- *  op1:
- *   op:
- *    opfield1: value1
- *   params:
- *    param2: value2
- *
- *  # neither op field nor params named, so all assumed to be op fields
- *  op2:
- *   opfield1: value1
- *   param2: value2
- *
- *  # in this case, if param2 is meant to be config level,
- *  # it is a likely config error that should be thrown to the user
- *  # only op field explicitly named, so remainder automatically pushed into params
- *  op3:
- *   op:
- *    opfield1: value1
- *    param2: value2
- *
- *  # only params explicitly named, so remainder pushed into op payload
- *  op4:
- *   params:
- *    param2: value2
- *   opfield1: value1
- * }

    - * - *

    - * All of these are considered valid constructions, and all of them may actually achieve the same result. - * This looks like an undesirable problem, but it serves to simplify things for users in one specific way: It allows - * them to be a vague, within guidelines, and still have a valid workload definition. - * The NoSQLBench runtime does a non-trivial amount of processing on the op template to - * ensure that it can conform to an unambiguous normalized internal structure on your behalf. - * This is needed because of how awful YAML is as a user configuration language in spite of its - * ubiquity in practice. The basic design guideline for these forms is that the op template must - * mean what a reasonable user would assume without looking at any documentation. - * - *

    - *

    Design Invariants

    - * - *

    The above rules imply invariants, which are made explicit here.

    - * - *

    Single Purpose Fields

    - *

    You may not use an op field name or parameter name for more than one purpose.

    - * - *

    Shared Namespace

    - *

    Treat all parameters supported by a driver adapter and it's op fields as a globally shared namespace, even - * if it is not.

    - * - *

    This avoids creating any confusion about what a parameter can be used for and how to use it for the right thing in - * the right place. For example, you may not use the parameter name `socket` in an op template to mean one thing and then use it - * at the driver adapter level to mean something different. However, if the meaning is congruent, a driver developer - * may choose to support some cross-cutting parameters at the activity level. These allowances are explicit, - * however, as each driver dictates what it will allow as activity parameters.

    - * - *

    Layered Resolution

    - * - *

    Users may specify op payload fields within op params or activity params as fallback config sources in that - * order.

    - * - *
      - *
    • IF a name is valid as an op field, it must also be valid as such when specified in op params.
      • - *
    • If a name is valid as an op field, it must also be valid as such when specified in activity params, within the - * scope of {@link ParsedOp}
      • - *
    • When an op field is found via op params or activity params, it may NOT be dynamic. If dynamic values are intended - * to be provided at a common layer in the workload, then bindings or template variables support this already.
      • - *
    - *
    2. - * - *

    Configuration Fields

    - * You must access non-payload params via Config-oriented methods. - *
      - *
    • Op Templates contain op payload data and op configs (params, activity params).
      • - *
    • You must use only {@link ParsedOp} getters with "...Config..." names, such as - * {@link #getConfigOr(String, Object, long)} - * when accessing non-payload fields.
      • - *
    • When a dynamic value is found via one of these calls, an error should be thrown, - * as configuration level data is not expected to be variant per-operation or cycle.
      • - *
    - * - * - *

    Sanity Checks

    - * - *

    The user must be warned when a required or optional config value is missing from op params (or activity - * params), but a value of the same name is found in op payload fields. - * If rule #1 is followed, and names are unambiguous across the driver, then it is almost certainly a configuration - * error.

    - * - *

    Precedence

    - *

    The order of precedence for values is: - *

      - *
    • op payload fields
      • - *
    • op param fields
      • - *
    • activity params (when enabled, see below)
      • - *
    - *

    - * - *

    Enabling Activity Params

    - *

    If a user wants to allow an activity param as an default for an fields, they must publish the op field - * name in the configuration model for the activity. Otherwise it is an error to specify the value at the activity - * level.

    - * - *
    - * - *

    Op Payload Forms

    - * Field values can come from multiple sources. These forms and any of their combinations are supported. - * - *

    Static Op Fields

    - * 
    {@code
- * op:
- *  field1: value1
- *  field2:
- *   map3:
- *    key4: value4
- *    map5:
- *     key6: value6
- *  field7: false
- *  field8: 8.8
- * }
    - *

    - * As shown, any literal value of any valid YAML type, including structured values like lists or maps are accepted as - * static op template values. A static value is any value which contains zero bind points at any level. - * - *

    Dynamic Op Fields with Binding References

    - * 
    {@code
- * op:
- * field1: "{binding1}"
- * field2: "value is: {binding1}"
- * }
- *
    - * - *

    In this form, {@code {binding1}} is known as a binding reference, since the binding function is defined - * elsewhere under the given name. The first field "field1" is specified with no leading nor trailing literals, and - * is thus taken as a raw binding reference, meaning that it will not be converted to a String. The second, - * named "field2", is what is known as a string template, and is syntactical sugar for a more complex binding - * which concatenates static and dynamic parts together. In this form, object types produced by binding functions are - * converted to string form before concatenation. - *

    - * - *

    Note that a raw {@code {binding1}} value (without quotes) would be NOT be a binding reference, since YAML - * is a superset of JSON. this means that {@code {binding1}} would be converted to a map or JSON object type - * with invalid contents. This is warned about when detected as a null valued map key, although it also makes - * null values invalid for ANY op template value.

    - * - *

    Dynamic Op Fields with Binding Definitions

    - * 
    {@code
- * op:
- *  field1: "{{NumberNameToString()}}"
- *  field2: "value is: {{NumberNameToString()}}"
- * }
- *
    - *

    - * This form has exactly the same effect as the previous example as long as your bindings definitions included: - * 

    {@code
- * bindings:
- *  binding1: NumberNameToString();
- * }
    - * - *

    Dynamic Op Fields with Structure

    - * - * 
    {@code
- * op:
- *  field1:
- *   k1: "{binding1}
- *   k2: "literal value"
- *  field2:
- *   - "value3"
- *   - "{binding4}"
- *   - "a value: {binding5}"
- *   - "{{NumberNameToString}}"
- *   - "a value: {{NumberNameToString()}}"
- * }
    - * - *

    This example combines the previous ones with structure and dynamic values. Both field1 and field2 are dynamic, - * since each contains some dynamic value or template within. When field1 is accessed within a cycle, that cycle's - * value - * will be used as the seed to generate equivalent structures with all the literal and dynamic elements inserted as - * the template implies. As before, direct binding references like {@code {binding4}} will be inserted into the - * structure with whatever type the binding definition produces, so if you want strings in them, ensure that you - * configure your binding definitions thusly.

    - * - *

    Op Template Params

    - * 
    {@code
- * params:
- *  prepared: true
- * ops:
- *  op1:
- *   field1: value1
- *   params:
- *    prepared: false
- * }
    - *

    - * The params section are the first layer of external configuration values that an op template can use to distinguish - * op configuration parameters from op payload content. - * Op Template Params are referenced when any of the {@link #getConfigOr(String, Object, long)} or other - * ...Config... - * getters are used (bypassing non-param fields). They are also accessed as a fallback when no static nor dynamic value - * is found - * for a reference op template field. Unlike op fields, op params cascade from within the workload YAML from the - * document level, - * down to each block and then down to each statement. - * - *

    Activity Params

    - * 
    {@code
- * ./nb run driver=... workload=... cl=LOCAL_QUORUM
- * }
    - * - *

    When explicitly allowed by a driver adapter's configuration model, values like {@code cl} above can be seen as - * another fallback source for configuration parameters. The {@link ParsedOp} implementation will automatically look - * in the activity parameters if needed to find a missing configuration parameter, but this will only work if - * the specific named parameter is allowed at the activity level.

    - * - *
    - *

    Alternate Names

    - *

    Sometimes you may need to support more than one name for the same purpose. In such cases, there are helper - * methods which can be used to reduce from a set of possible field names to a single one. - *

      - *
    • {@link #requiredFieldOf(String...)} and {@link #requiredFieldOf(List)} will find exactly one field within the set of - * possible fields. - * Zero or more than one will throw an error.
      • - *
    • {@link #optionalFieldOf(String...)} and {@link #optionalFieldOf(List)} will find exactly zero or one - * field within the set of possible fields. More than one will throw an error.
      • - *
    - *

    - */ +

    ParsedOp API

    + +

    This is the primary developer-focused API for driver developers to use when up-converting op + templates + to operations. This {@link ParsedOp} is a wrapper around the op template structure. It provides + many + ways of constructing higher-order objects from a variety of sources.

    + +

    Supporting Variety

    +

    + For some drivers or protocols, the primary user interface is a statement format or grammar. For CQL + or SQL, + the most natural way of describing templates for operations is in that native format. For others, an + operation + template may look like a block of JSON or a HTTP request. All these forms are supported. In order to + deal with + the variety, there is a set of detailed rules for how the workload definitions are transformed into + driver + operations for a native driver. The high-level flow is: +

      +
    1. Op Template Form
      2. +
    2. Normalized Data Structure
      3. +
    3. ParsedOp Form
      4. +
    +

    + The specifications govern how the raw op template form is transformed into the normalized data + structure. + This documentation focuses on the API provided by the last form, this class, although peripheral + awareness + of the first two forms will certainly help for any advanced scenarios. You can find detailed + examples and + specification tests under the workload_definition folder of the adapters-api module. +

    + +
    +

    Op Template Parsing

    +
      +
    1. Rule #1: All op templates are parsed into an internal normalized structure which contains: +
        +
      1. A map of op fields, which can consist of: +
          +
        1. static field values
          2. +
        2. dynamic field values, the actual value of which can only be known for a given cycle
          3. +
        +
        2. +
      2. Access to auxiliary configuration values like activity parameters. These can back-fill + when values aren't present in the direct static or dynamic op fields.
        3. + + +
      3. Rule #2: When asking for a dynamic parameter, static parameters may be automatically promoted to + functional form + as a back-fill.
        4. +
      4. Rule #3: When asking for static parameters, config parameters may automatically be promoted as a + back-fill.
        5. +
      + The net effect of these rules is that the NoSQLBench driver developer may safely use functional + forms to access data + in the op template, or may decide that certain op fields must only be provided in a static way per + operation. +

      +
      +

      Distinguishing Op Payload from Op Config

      +

      When a user specifies an op template, they may choose to provide only a single set of op fields + without + distinguishing between config or payload, or they may choose to directly configure each. + + 

      {@code
+ops:
+
+# both op and params explicitly named
+op1:
+op:
+opfield1: value1
+params:
+param2: value2
+
+# neither op field nor params named, so all assumed to be op fields
+op2:
+opfield1: value1
+param2: value2
+
+# in this case, if param2 is meant to be config level,
+# it is a likely config error that should be thrown to the user
+# only op field explicitly named, so remainder automatically pushed into params
+op3:
+op:
+opfield1: value1
+param2: value2
+
+# only params explicitly named, so remainder pushed into op payload
+op4:
+params:
+param2: value2
+opfield1: value1
+}

      + +

      + All of these are considered valid constructions, and all of them may actually achieve the same + result. + This looks like an undesirable problem, but it serves to simplify things for users in one specific + way: It allows + them to be a vague, within guidelines, and still have a valid workload definition. + The NoSQLBench runtime does a non-trivial amount of processing on the op template to + ensure that it can conform to an unambiguous normalized internal structure on your behalf. + This is needed because of how awful YAML is as a user configuration language in spite of its + ubiquity in practice. The basic design guideline for these forms is that the op template must + mean what a reasonable user would assume without looking at any documentation. + +

      +

      Design Invariants

      + +

      The above rules imply invariants, which are made explicit here.

      + +

      Single Purpose Fields

      +

      You may not use an op field name or parameter name for more than one purpose.

      + +

      Shared Namespace

      +

      Treat all parameters supported by a driver adapter and it's op fields as a globally shared + namespace, even + if it is not.

      + +

      This avoids creating any confusion about what a parameter can be used for and how to use it for + the right thing in + the right place. For example, you may not use the parameter name `socket` in an op template to mean + one thing and then use it + at the driver adapter level to mean something different. However, if the meaning is congruent, a + driver developer + may choose to support some cross-cutting parameters at the activity level. These allowances are + explicit, + however, as each driver dictates what it will allow as activity parameters.

      + +

      Layered Resolution

      + +

      Users may specify op payload fields within op params or activity params as fallback config + sources in that + order.

      + +
        +
      • IF a name is valid as an op field, it must also be valid as such when specified in op + params.
        • +
      • If a name is valid as an op field, it must also be valid as such when specified in activity + params, within the + scope of {@link ParsedOp}
        • +
      • When an op field is found via op params or activity params, it may NOT be dynamic. If dynamic + values are intended + to be provided at a common layer in the workload, then bindings or template variables support this + already.
        • +
      +
      2. + +

      Configuration Fields

      + You must access non-payload params via Config-oriented methods. +
        +
      • Op Templates contain op payload data and op configs (params, activity params).
        • +
      • You must use only {@link ParsedOp} getters with "...Config..." names, such as + {@link #getConfigOr(String, Object, long)} + when accessing non-payload fields.
        • +
      • When a dynamic value is found via one of these calls, an error should be thrown, + as configuration level data is not expected to be variant per-operation or cycle.
        • +
      + + +

      Sanity Checks

      + +

      The user must be warned when a required or optional config value is missing from op params + (or activity + params), but a value of the same name is found in op payload fields. + If rule #1 is followed, and names are unambiguous across the driver, then it is almost certainly a + configuration + error.

      + +

      Precedence

      +

      The order of precedence for values is: +

        +
      • op payload fields
        • +
      • op param fields
        • +
      • activity params (when enabled, see below)
        • +
      +

      + +

      Enabling Activity Params

      +

      If a user wants to allow an activity param as an default for an fields, they must publish the op + field + name in the configuration model for the activity. Otherwise it is an error to specify the value at + the activity + level.

      + +
      + +

      Op Payload Forms

      + Field values can come from multiple sources. These forms and any of their combinations are + supported. + +

      Static Op Fields

      + 
      {@code
+op:
+field1: value1
+field2:
+map3:
+key4: value4
+map5:
+key6: value6
+field7: false
+field8: 8.8
+}
      +

      + As shown, any literal value of any valid YAML type, including structured values like lists or maps + are accepted as + static op template values. A static value is any value which contains zero bind points at any + level. + +

      Dynamic Op Fields with Binding References

      + 
      {@code
+op:
+field1: "{binding1}"
+field2: "value is: {binding1}"
+}
+
      + +

      In this form, {@code {binding1}} is known as a binding reference, since the binding + function is defined + elsewhere under the given name. The first field "field1" is specified with no leading nor trailing + literals, and + is thus taken as a raw binding reference, meaning that it will not be converted to a + String. The second, + named "field2", is what is known as a string template, and is syntactical sugar for a more + complex binding + which concatenates static and dynamic parts together. In this form, object types produced by binding + functions are + converted to string form before concatenation. +

      + +

      Note that a raw {@code {binding1}} value (without quotes) would be NOT be a binding reference, + since YAML + is a superset of JSON. this means that {@code {binding1}} would be converted to a map or JSON object + type + with invalid contents. This is warned about when detected as a null valued map key, although it also + makes + null values invalid for ANY op template value.

      + +

      Dynamic Op Fields with Binding Definitions

      + 
      {@code
+op:
+field1: "{{NumberNameToString()}}"
+field2: "value is: {{NumberNameToString()}}"
+}
+
      +

      + This form has exactly the same effect as the previous example as long as your bindings definitions + included: + 

      {@code
+bindings:
+binding1: NumberNameToString();
+}
      + +

      Dynamic Op Fields with Structure

      + + 
      {@code
+op:
+field1:
+k1: "{binding1}
+k2: "literal value"
+field2:
+- "value3"
+- "{binding4}"
+- "a value: {binding5}"
+- "{{NumberNameToString}}"
+- "a value: {{NumberNameToString()}}"
+}
      + +

      This example combines the previous ones with structure and dynamic values. Both field1 and field2 + are dynamic, + since each contains some dynamic value or template within. When field1 is accessed within a cycle, + that cycle's + value + will be used as the seed to generate equivalent structures with all the literal and dynamic elements + inserted as + the template implies. As before, direct binding references like {@code {binding4}} will be inserted + into the + structure with whatever type the binding definition produces, so if you want strings in them, ensure + that you + configure your binding definitions thusly.

      + +

      Op Template Params

      + 
      {@code
+params:
+prepared: true
+ops:
+op1:
+field1: value1
+params:
+prepared: false
+}
      +

      + The params section are the first layer of external configuration values that an op template can use + to distinguish + op configuration parameters from op payload content. + Op Template Params are referenced when any of the {@link #getConfigOr(String, Object, long)} or + other + ...Config... + getters are used (bypassing non-param fields). They are also accessed as a fallback when no static + nor dynamic value + is found + for a reference op template field. Unlike op fields, op params cascade from within the workload YAML + from the + document level, + down to each block and then down to each statement. + +

      Activity Params

      + 
      {@code
+./nb run driver=... workload=... cl=LOCAL_QUORUM
+}
      + +

      When explicitly allowed by a driver adapter's configuration model, values like {@code cl} above + can be seen as + another fallback source for configuration parameters. The {@link ParsedOp} implementation will + automatically look + in the activity parameters if needed to find a missing configuration parameter, but this will only + work if + the specific named parameter is allowed at the activity level.

      + +
      +

      Alternate Names

      +

      Sometimes you may need to support more than one name for the same purpose. In such cases, there + are helper + methods which can be used to reduce from a set of possible field names to a single one. +

        +
      • {@link #requiredFieldOf(String...)} and {@link #requiredFieldOf(List)} will find exactly one + field within the set of + possible fields. + Zero or more than one will throw an error.
        • +
      • {@link #optionalFieldOf(String...)} and {@link #optionalFieldOf(List)} will find exactly zero or + one + field within the set of possible fields. More than one will throw an error.
        • +
      +

      */ public class ParsedOp extends NBBaseComponent implements LongFunction>, NBComponent, StaticFieldReader, DynamicFieldReader { private static final Logger logger = LogManager.getLogger(ParsedOp.class); /** - * The names of payload values in the result of the operation which should be saved. - * The keys in this map represent the name of the value as it would be found in the native - * representation of a result. If the values are defined, then each one represents the name - * that the found value should be saved as instead of the original name. + The names of payload values in the result of the operation which should be saved. + The keys in this map represent the name of the value as it would be found in the native + representation of a result. If the values are defined, then each one represents the name + that the found value should be saved as instead of the original name. */ private final List captures = new ArrayList<>(); @@ -339,55 +396,58 @@ public class ParsedOp extends NBBaseComponent implements LongFunction, Map>> preprocessors; /** - * Create a parsed command from an Op template. This version is exactly like - * except that it allows - * preprocessors. Preprocessors are all applied to the the op template before - * it is applied to the parsed command fields, allowing you to combine or destructure - * fields from more tha one representation into a single canonical representation - * for processing. - * - * @param opTemplate - * The OpTemplate as provided by a user via YAML, JSON, or API (data structure) - * @param activityCfg - * The activity configuration, used to resolve nested config parameters - * @param preprocessors - * Map->Map transformers. + Create a parsed command from an Op template. This version is exactly like + except that it allows + preprocessors. Preprocessors are all applied to the the op template before + it is applied to the parsed command fields, allowing you to combine or destructure + fields from more tha one representation into a single canonical representation + for processing. + @param opTemplate + The OpTemplate as provided by a user via YAML, JSON, or API (data structure) + @param activityCfg + The activity configuration, used to resolve nested config parameters + @param preprocessors + Map->Map transformers. */ + public ParsedOp( - OpTemplate opTemplate, - NBConfiguration activityCfg, + OpTemplate opTemplate, NBConfiguration activityCfg, List, Map>> preprocessors, NBComponent parent ) { - super(parent, NBLabels.forKV(((parent instanceof ParsedOp) ? "subop" : "op"), opTemplate.getName())); + super( + parent, + NBLabels.forKV(((parent instanceof ParsedOp) ? "subop" : "op"), opTemplate.getName()) + ); this._opTemplate = opTemplate; this.activityCfg = activityCfg; - Map map = opTemplate.getOp().orElseThrow(() -> - new OpConfigError("ParsedOp constructor requires a non-null value for the op field, but it was missing.")); + Map map = opTemplate.getOp().orElseThrow(() -> new OpConfigError( + "ParsedOp constructor requires a non-null value for the op field, but it was missing.")); + this.preprocessors = preprocessors; for (Function, Map> preprocessor : preprocessors) { map = preprocessor.apply(map); } this.tmap = new ParsedTemplateMap( - getName(), - map, - opTemplate.getBindings(), - List.of(opTemplate.getParams(), - activityCfg.getMap()) + getName(), map, opTemplate.getBindings(), + List.of(opTemplate.getParams(), activityCfg.getMap()) ); - NBLabels opLabels = parent.getLabels().and((parent instanceof ParsedOp) ? "subop" : "op", this.getName()); + NBLabels opLabels = parent.getLabels().and( + (parent instanceof ParsedOp) ? "subop" : "op", this.getName()); if (tmap.isStatic("labels")) { Object labelSpecObject = tmap.takeStaticValue("labels", Object.class); if (labelSpecObject instanceof String labelsSpec) { NBLabels op_specific_labels = NBLabelSpec.parseLabels(labelsSpec); opLabels = opLabels.and(op_specific_labels); } else { - throw new OpConfigError("parsing labels as type '" + labelSpecObject.getClass().getSimpleName() + "' is not supported."); + throw new OpConfigError( + "parsing labels as type '" + labelSpecObject.getClass().getSimpleName() + "' is not supported."); } } else if (tmap.isDynamic("labels")) { throw new OpConfigError("Labels may not be dynamic."); @@ -395,6 +455,14 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The parameter type of the return type, used at compile time only to qualify asserted return type - * @return A value of type T, or null + Get the static value for the provided name, cast to the required type. + @param field + Name of the field to get + @param classOfT + The type of the field to return. If actual type is not compatible to a cast to this type, + then a + casting error will be thrown. + @param + The parameter type of the return type, used at compile time only to qualify asserted return + type + @return A value of type T, or null */ @Override public T getStaticValue(String field, Class classOfT) { @@ -460,14 +529,14 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The parameter type of the return type. used at compile time only to quality return type. - * @return A value of type T, or null + Get the static value for the provided name, cast to the required type, where the type is + inferred + from the calling context. + @param field + Name of the field to get + @param + The parameter type of the return type. used at compile time only to quality return type. + @return A value of type T, or null */ @Override public T getStaticValue(String field) { @@ -483,18 +552,17 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The type of the field to return. - * @return The value - * @throws RuntimeException - * if the field name is only present in the dynamic fields. + Get the named static field value, or return the provided default, but throw an exception if + the named field is dynamic. + @param name + The name of the field value to return. + @param defaultValue + A value to return if the named value is not present in static nor dynamic fields. + @param + The type of the field to return. + @return The value + @throws RuntimeException + if the field name is only present in the dynamic fields. */ @Override public T getStaticValueOr(String name, T defaultValue) { @@ -502,37 +570,36 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The type of the value to return - * @return A configuration value - * @throws NBConfigError - * if the named field is defined dynamically, - * as in this case, it is presumed that the parameter is not supported unless it is defined statically. + Get the specified parameter by the user using the defined field which is closest to the op + template. This is the standard way of getting parameter values which can be specified at the + op template, op param, or activity level. + @param name + The name of the configuration param + @param defaultValue + the default value to return if the value is not defined anywhere in + (op fields, op params, activity params) + @param + The type of the value to return + @return A configuration value + @throws NBConfigError + if the named field is defined dynamically, + as in this case, it is presumed that the parameter is not supported unless it is defined + statically. */ public T getStaticConfigOr(String name, T defaultValue) { return tmap.getStaticConfigOr(name, defaultValue); } /** - * Get the parameter value from a static op template field OR any of the provided optional sources - * of op template values, including the activity parameters - * - * @param name - * The config field name - * @param defaultValue - * The default value, if the field is not defined in the op template nor the activity params - * @param - * The type of the field - * @return The config value. + Get the parameter value from a static op template field OR any of the provided optional sources + of op template values, including the activity parameters + @param name + The config field name + @param defaultValue + The default value, if the field is not defined in the op template nor the activity params + @param + The type of the field + @return The config value. */ public T takeStaticConfigOr(String name, T defaultValue) { return tmap.takeStaticConfigOr(name, defaultValue); @@ -549,9 +616,9 @@ public class ParsedOp extends NBBaseComponent implements LongFunction T getConfigOr(String name, T defaultValue, long input) { return tmap.getConfigOr(name, defaultValue, input); @@ -559,16 +626,16 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The parameter type of the return - * @return An optional value, empty unless the named value is defined in the static field map. + Return an optional value for the named field. This is an {@link Optional} form of + {@link #getStaticValue}. + @param field + Name of the field to get + @param classOfT + The type of field to return. If the actual type is not compatible to a cast to this type, + then a casting error will be thrown. + @param + The parameter type of the return + @return An optional value, empty unless the named value is defined in the static field map. */ @Override public Optional getOptionalStaticValue(String field, Class classOfT) { @@ -581,18 +648,19 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The parameter type of the returned value. Inferred from usage context. - * @return The value. + Get the named field value for a given long input. This uses parameter type inference -- The + casting + to the return type will be based on the type of any assignment or casting on the caller's side. + Thus, if the actual type is not compatable to a cast to the needed return type, a casting error + will + be thrown. + @param field + The name of the field to get. + @param input + The seed value, or cycle value for which to generate the value. + @param + The parameter type of the returned value. Inferred from usage context. + @return The value. */ @Override public T get(String field, long input) { @@ -600,32 +668,30 @@ public class ParsedOp extends NBBaseComponent implements LongFunction getDefinedNames() { return tmap.getOpFieldNames(); } /** - * Get the op field as a {@link LongFunction} of String. This is a convenience form for - * {@link #getAsRequiredFunction(String, Class)} - * - * @param name - * The field name which must be defined as static or dynamic - * @return A function which can provide the named field value + Get the op field as a {@link LongFunction} of String. This is a convenience form for + {@link #getAsRequiredFunction(String, Class)} + @param name + The field name which must be defined as static or dynamic + @return A function which can provide the named field value */ public LongFunction getAsRequiredFunction(String name) { return tmap.getAsRequiredFunction(name, String.class); } /** - * Get the op field as a {@link LongFunction} - * - * @param name - * The field name which must be defined as static or dynamic - * @param type - * The value type which the field must be assignable to - * @return A function which can provide a value for the given name and type + Get the op field as a {@link LongFunction} + @param name + The field name which must be defined as static or dynamic + @param type + The value type which the field must be assignable to + @return A function which can provide a value for the given name and type */ public Optional> getAsOptionalFunction(String name, Class type) { return tmap.getAsOptionalFunction(name, type); @@ -648,16 +714,16 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The type of value to return - * @return A {@link LongFunction} of type V + Get a LongFunction which returns either the static value, the dynamic value, or the default + value, + in that order, depending on where it is found first. + @param name + The param name for the value + @param defaultValue + The default value to provide the value is not defined for static nor dynamic + @param + The type of value to return + @return A {@link LongFunction} of type V */ @Override public LongFunction getAsFunctionOr(String name, V defaultValue) { @@ -665,28 +731,30 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * The type of object to return - * @return A caching function which chains to the init function, with caching + Get a LongFunction that first creates a LongFunction of String as in + {@link #getAsRequiredFunction(String, Class)}, but then + applies the result and caches it for subsequent access. This relies on {@link ObjectCache} + internally. + @param fieldname + The name of the field which could contain a static or dynamic value + @param defaultValue + The default value to use in the init function if the fieldname is not defined as static nor + dynamic + @param init + A function to apply to the value to produce the product type + @param + The type of object to return + @return A caching function which chains to the init function, with caching */ - public LongFunction getAsCachedFunctionOr(String fieldname, String defaultValue, Function init) { + public LongFunction getAsCachedFunctionOr( + String fieldname, String defaultValue, Function init) { return tmap.getAsCachedFunctionOr(fieldname, defaultValue, init); } /** - * @param field - * The requested field name - * @return true if the named field is defined as static or dynamic + @param field + The requested field name + @return true if the named field is defined as static or dynamic */ @Override public boolean isDefined(String field) { @@ -694,23 +762,23 @@ public class ParsedOp extends NBBaseComponent implements LongFunction type) { @@ -718,83 +786,82 @@ public class ParsedOp extends NBBaseComponent implements LongFunction> newListBinder(String... fields) { return tmap.newListBinder(fields); } /** - * @param fields - * The ordered field names for which the {@link ListBinder} will be created - * @return a new {@link ListBinder} which can produce a {@link List} of Objects from a long input. + @param fields + The ordered field names for which the {@link ListBinder} will be created + @return a new {@link ListBinder} which can produce a {@link List} of Objects from a long input. */ public LongFunction> newListBinder(List fields) { return tmap.newListBinder(fields); } /** - * @param fields - * The ordered field names for which the {@link OrderedMapBinder} will be created - * @return a new {@link OrderedMapBinder} which can produce a {@link Map} of String to Objects from a long input. + @param fields + The ordered field names for which the {@link OrderedMapBinder} will be created + @return a new {@link OrderedMapBinder} which can produce a {@link Map} of String to Objects from + a long input. */ public LongFunction> newOrderedMapBinder(String... fields) { return tmap.newOrderedMapBinder(fields); } /** - * @param fields - * The ordered field names for which the {@link ArrayBinder} will be created - * @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. + @param fields + The ordered field names for which the {@link ArrayBinder} will be created + @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. */ public LongFunction newArrayBinder(String... fields) { return tmap.newArrayBinder(fields); } /** - * @param fields - * The ordered field names for which the {@link ArrayBinder} will be created - * @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. + @param fields + The ordered field names for which the {@link ArrayBinder} will be created + @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. */ public LongFunction newArrayBinder(List fields) { return tmap.newArrayBinder(fields); } /** - * @param bindPoints - * The {@link BindPoint}s for which the {@link ArrayBinder} will be created - * @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. + @param bindPoints + The {@link BindPoint}s for which the {@link ArrayBinder} will be created + @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. */ public LongFunction newArrayBinderFromBindPoints(List bindPoints) { return tmap.newArrayBinderFromBindPoints(bindPoints); } /** - * Get the {@link LongFunction} which is used to resolve a dynamic field value. - * - * @param field - * The field name for a dynamic parameter - * @return The mapping function + Get the {@link LongFunction} which is used to resolve a dynamic field value. + @param field + The field name for a dynamic parameter + @return The mapping function */ public LongFunction getMapper(String field) { return tmap.getMapper(field); } /** - * @return the logical map size, including all static and dynamic fields + @return the logical map size, including all static and dynamic fields */ public int getSize() { return tmap.getSize(); @@ -805,113 +872,119 @@ public class ParsedOp extends NBBaseComponent implements LongFunction - * Generic type for the enum class - * @return Optionally, an enum value which matches, or {@link Optional#empty()} - * @throws OpConfigError - * if more than one field matches + Given an enum of any type, return the enum value which is found in any of the field names of the + op template, + ignoring case and any non-word characters. This is useful for matching op templates to op types + where the + presence + of a field determines the type. Further, if there are multiple matching names, an + {@link OpConfigError} is thrown + to + avoid possible ambiguity. + @param enumclass + The enum class for matching values + @param + Generic type for the enum class + @return Optionally, an enum value which matches, or {@link Optional#empty()} + @throws OpConfigError + if more than one field matches */ - public , V> Optional> getTypeAndTargetFromEnum(Class enumclass, Class valueClass) { + public , V> Optional> getTypeAndTargetFromEnum( + Class enumclass, Class valueClass) { return tmap.getOptionalTargetEnum(enumclass, valueClass); } public , V> Optional> getOptionalTypeAndTargetEnum( - Class enumclass, - Class valueClass - ) { + Class enumclass, Class valueClass) { return tmap.getOptionalTargetEnum(enumclass, valueClass); } public , V> Optional> getOptionalTypeAndTargetEnum( - Class enumclass, - Class valueClass, - String alternateTypeField, + Class enumclass, Class valueClass, String alternateTypeField, String alternateValueField ) { - return tmap.getOptionalTargetEnum(enumclass, valueClass, alternateTypeField, alternateValueField); + return tmap.getOptionalTargetEnum( + enumclass, valueClass, alternateTypeField, alternateValueField); } - public , V> TypeAndTarget getTypeAndTarget(Class enumclass, Class valueClass) { + public , V> TypeAndTarget getTypeAndTarget( + Class enumclass, Class valueClass) { return tmap.getTargetEnum(enumclass, valueClass); } - public , V> TypeAndTarget getTypeAndTarget(Class enumclass, Class valueclass, String tname, String vname) { + public , V> TypeAndTarget getTypeAndTarget( + Class enumclass, Class valueclass, String tname, String vname) { return tmap.getTargetEnum(enumclass, valueclass, tname, vname); } - public > Optional getOptionalEnumFromField(Class enumclass, String fieldName) { + public > Optional getOptionalEnumFromField( + Class enumclass, String fieldName) { return tmap.getOptionalEnumFromField(enumclass, fieldName); } - public > E getEnumFromFieldOr(Class enumClass, E defaultEnum, String fieldName) { + public > E getEnumFromFieldOr( + Class enumClass, E defaultEnum, String fieldName) { return getOptionalEnumFromField(enumClass, fieldName).orElse(defaultEnum); } + public > E takeEnumFromFieldOr( + Class enumClass, E defaultEnum, String fieldName) { + return tmap.takeOptionalEnumFromField(enumClass, fieldName).orElse(defaultEnum); + } + + /** - *

      Enhance a {@link Function} with another required named field or function combiner OR a default - * value.

      - * - * @param func - * The base function - * @param field - * The field name to derive the named enhancer function from - * @param type - * The type of the field value - * @param defaultFe - * The default value of the field, if none is provided - * @param combiner - * A {@link BiFunction} which applies the field or function combiner to the base function - * @param - * The base function result type - * @param - * The enhancer function result type - * @return an enhanced function +

      Enhance a {@link Function} with another required named field or function combiner OR + a default + value.

      + @param func + The base function + @param field + The field name to derive the named enhancer function from + @param type + The type of the field value + @param defaultFe + The default value of the field, if none is provided + @param combiner + A {@link BiFunction} which applies the field or function combiner to the base function + @param + The base function result type + @param + The enhancer function result type + @return an enhanced function */ public LongFunction enhanceDefaultFunc( - LongFunction func, - String field, - Class type, - FE defaultFe, + LongFunction func, String field, + Class type, FE defaultFe, BiFunction combiner ) { LongFunction fieldEnhancerFunc = getAsFunctionOr(field, defaultFe); LongFunction faLongFunction = func; - LongFunction lfa = l -> combiner.apply(faLongFunction.apply(l), fieldEnhancerFunc.apply(l)); + LongFunction lfa = l -> combiner.apply( + faLongFunction.apply(l), fieldEnhancerFunc.apply(l)); return lfa; } /** - *

      Enhance a {@link Function} with a named required function, or throw an error.

      - * - * @param func - * The base function - * @param field - * The field name to derive the named enhancer function from - * @param type - * The type of the field value - * @param combiner - * A {@link BiFunction} which applies the field or function combiner to the base function - * @param - * The base function result type - * @param - * The enhancer function result type - * @return a version of the base function, optionally enhanced +

      Enhance a {@link Function} with a named required function, or throw an error.

      + @param func + The base function + @param field + The field name to derive the named enhancer function from + @param type + The type of the field value + @param combiner + A {@link BiFunction} which applies the field or function combiner to the base function + @param + The base function result type + @param + The enhancer function result type + @return a version of the base function, optionally enhanced */ public LongFunction enhanceFunc( - LongFunction func, - String field, - Class type, - BiFunction combiner + LongFunction func, String field, + Class type, BiFunction combiner ) { LongFunction fieldEnhancerFunc = getAsRequiredFunction(field, type); LongFunction lfa = l -> combiner.apply(func.apply(l), fieldEnhancerFunc.apply(l)); @@ -919,25 +992,23 @@ public class ParsedOp extends NBBaseComponent implements LongFunctionEnhance a {@link Function} with a named optional function IFF it exists.

      - * - * @param func - * The base function - * @param field - * The field name to derive the named enhancer function from - * @param type - * The type of the field value - * @param combiner - * A {@link BiFunction} which applies the field or function combiner to the base function - * @param - * The base function result type - * @param - * The enhancer function result type - * @return a version of the base function, optionally enhanced +

      Enhance a {@link Function} with a named optional function IFF it exists.

      + @param func + The base function + @param field + The field name to derive the named enhancer function from + @param type + The type of the field value + @param combiner + A {@link BiFunction} which applies the field or function combiner to the base function + @param + The base function result type + @param + The enhancer function result type + @return a version of the base function, optionally enhanced */ public LongFunction enhanceFuncOptionally( - LongFunction func, - String field, + LongFunction func, String field, Class type, BiFunction combiner ) { @@ -956,15 +1027,17 @@ public class ParsedOp extends NBBaseComponent implements LongFunction opfields, SubOpNaming naming) { + private ParsedOp makeSubOp( + String fromOpField, String elemName, Map opfields, SubOpNaming naming) { String subkey = opfields.containsKey("name") ? opfields.get("name").toString() : elemName; String subopName = switch (naming) { case SubKey -> subkey; @@ -973,17 +1046,10 @@ public class ParsedOp extends NBBaseComponent implements LongFunction(_opTemplate.getTags()) {{ - put("subop", subopName); - }}, - _opTemplate.getBindings(), - _opTemplate.getParams(), - opfields - ), - this.activityCfg, - List.of(), - this + subopName, new LinkedHashMap(_opTemplate.getTags()) {{ + put("subop", subopName); + }}, _opTemplate.getBindings(), _opTemplate.getParams(), opfields, 100 + ), this.activityCfg, List.of(), this ); } @@ -996,14 +1062,16 @@ public class ParsedOp extends NBBaseComponent implements LongFunction raw.get(name)).orElseThrow(() -> - new OpConfigError("Could not find op field '" + name + "' for subop on parent op '" + name + "'")); + Object o = _opTemplate.getOp().map(raw -> raw.get(name)).orElseThrow( + () -> new OpConfigError( + "Could not find op field '" + name + "' for subop on parent op '" + name + "'")); if (o instanceof Map map) { return makeSubOp(this.getName(), name, map, naming); } else { @@ -1038,27 +1109,24 @@ public class ParsedOp extends NBBaseComponent implements LongFunctionEnhance an {@link Optional} {@link Function} with an optional named field or value combiner, - * IFF both functions are defined.

      - * - * @param func - * The base function - * @param field - * The field name to derive the named enhancer function from - * @param type - * The type of the field value - * @param combiner - * A {@link BiFunction} which applies the field or function combiner to the base function - * @param - * The base function result type - * @param - * The enhancer function result type - * @return the enhanced optional function +

      Enhance an {@link Optional} {@link Function} with an optional named field or value combiner, + IFF both functions are defined.

      + @param func + The base function + @param field + The field name to derive the named enhancer function from + @param type + The type of the field value + @param combiner + A {@link BiFunction} which applies the field or function combiner to the base function + @param + The base function result type + @param + The enhancer function result type + @return the enhanced optional function */ public Optional> enhanceOptionalFuncOptionally( - Optional> func, - String field, - Class type, + Optional> func, String field, Class type, BiFunction combiner ) { Optional> fieldEnhancerFunc = getAsOptionalFunction(field, type); @@ -1067,39 +1135,37 @@ public class ParsedOp extends NBBaseComponent implements LongFunction faLongFunction = func.get(); LongFunction feLongFunction = fieldEnhancerFunc.get(); - LongFunction lfa = l -> combiner.apply(faLongFunction.apply(l), feLongFunction.apply(l)); + LongFunction lfa = l -> combiner.apply( + faLongFunction.apply(l), feLongFunction.apply(l)); return Optional.of(lfa); } /** - *

      Enhance an {@link Optional} {@link Function} with a named field or function combiner OR a default value, - * IFF the base function is present.

      - * - *

      Create a required function for the specified field and default value, IFF the - * main function is present. The base type of the function remains the same, and if present, - * will be extended with the required field value or function in the provided combiner.

      - * - * @param func - * The base function - * @param field - * The field name to derive the named enhancer function from - * @param type - * The type of the field value - * @param defaultFe - * The default value of the field, if none is provided - * @param combiner - * A {@link BiFunction} which applies the field or function combiner to the base function - * @param - * The base function result type - * @param - * The enhancer function result type - * @return the enhanced optional base function +

      Enhance an {@link Optional} {@link Function} with a named field or function combiner OR a + default value, + IFF the base function is present.

      + +

      Create a required function for the specified field and default value, IFF the + main function is present. The base type of the function remains the same, and if present, + will be extended with the required field value or function in the provided combiner.

      + @param func + The base function + @param field + The field name to derive the named enhancer function from + @param type + The type of the field value + @param defaultFe + The default value of the field, if none is provided + @param combiner + A {@link BiFunction} which applies the field or function combiner to the base function + @param + The base function result type + @param + The enhancer function result type + @return the enhanced optional base function */ public Optional> enhanceOptionalDefaultFunc( - Optional> func, - String field, - Class type, - FE defaultFe, + Optional> func, String field, Class type, FE defaultFe, BiFunction combiner ) { if (func.isEmpty()) { @@ -1107,34 +1173,30 @@ public class ParsedOp extends NBBaseComponent implements LongFunction fieldEnhancerFunc = getAsFunctionOr(field, defaultFe); LongFunction faLongFunction = func.get(); - LongFunction lfa = l -> combiner.apply(faLongFunction.apply(l), fieldEnhancerFunc.apply(l)); + LongFunction lfa = l -> combiner.apply( + faLongFunction.apply(l), fieldEnhancerFunc.apply(l)); return Optional.of(lfa); } /** - *

      Enhance a {@link Function} with an optional enum function IFF it is defined.

      - * - * @param func - * The base function - * @param field - * The field name to derive the named enhancer function from - * @param type - * The type of the field value - * @param combiner - * A {@link BiFunction} which applies the field or function combiner to the base function - * @param - * The base function result type - * @param - * The enhancer function result type - * @return an (optionally) enhanced base function +

      Enhance a {@link Function} with an optional enum function IFF it is defined.

      + @param func + The base function + @param field + The field name to derive the named enhancer function from + @param type + The type of the field value + @param combiner + A {@link BiFunction} which applies the field or function combiner to the base function + @param + The base function result type + @param + The enhancer function result type + @return an (optionally) enhanced base function */ public > LongFunction enhanceEnumOptionally( - LongFunction func, - String field, - Class type, - BiFunction combiner - ) { + LongFunction func, String field, Class type, BiFunction combiner) { Optional> fieldEnhancerFunc = getAsOptionalEnumFunction(field, type); if (fieldEnhancerFunc.isEmpty()) { return func; @@ -1143,7 +1205,8 @@ public class ParsedOp extends NBBaseComponent implements LongFunction { S initialResult = func.apply(l); FE enumValue = feLongFunction.apply(l); - return combiner.apply(initialResult, enumValue); // Apply combiner and return the result + return combiner.apply( + initialResult, enumValue); // Apply combiner and return the result }; // LongFunction lfa = l -> combiner.apply(func.apply(l), feLongFunction.apply(l)); @@ -1163,7 +1226,7 @@ public class ParsedOp extends NBBaseComponent implements LongFunction getCaptures() { + public CapturePoints getCaptures() { return tmap.getCaptures(); } @@ -1177,11 +1240,7 @@ public class ParsedOp extends NBBaseComponent implements LongFunction LongFunction enhanceFuncOptionally2( - LongFunction func, - String field, - Class type, - String field2, - Class type2, + LongFunction func, String field, Class type, String field2, Class type2, TriFunction triCombiner ) { Optional> enhancer1 = getAsOptionalFunction(field, type); @@ -1201,14 +1260,14 @@ public class ParsedOp extends NBBaseComponent implements LongFunction LongFunction enhanceFuncPivot( - LongFunction func, - String field, + LongFunction func, String field, Class type, BiFunction combiner ) { Optional> fieldEnhancerFunc = getAsOptionalFunction(field, type); if (fieldEnhancerFunc.isEmpty()) { - throw new RuntimeException("required function enhancer field '" + field + " was not found in op:" + this); + throw new RuntimeException( + "required function enhancer field '" + field + " was not found in op:" + this); } LongFunction feLongFunction = fieldEnhancerFunc.get(); LongFunction lfa = l -> combiner.apply(func.apply(l), feLongFunction.apply(l)); @@ -1216,7 +1275,6 @@ public class ParsedOp extends NBBaseComponent implements LongFunction fields) { int count = 0; String found = null; @@ -1225,7 +1283,8 @@ public class ParsedOp extends NBBaseComponent implements LongFunction optionalFieldOf(List fields) { return Optional.ofNullable(oneFieldOrNull(fields)); } + public Optional optionalFieldOf(String... fields) { return Optional.ofNullable(oneFieldOrNull(List.of(fields))); } + public String requiredFieldOf(String... fields) { return requiredFieldOf(List.of(fields)); } + public String requiredFieldOf(List fields) { String field = oneFieldOrNull(fields); - if (field==null) { - throw new OpConfigError(STR."no fields were found with name in (\{fields}) for op template \{this}"); + if (field == null) { + throw new OpConfigError( + "no fields were found with name in (" + fields + ") for op template " + this); } return field; } diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/00_workload_specification.md b/nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/00_workload_specification.md rename to nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/01_spectest_formatting.md b/nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/01_spectest_formatting.md rename to nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/02_workload_structure.md b/nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/02_workload_structure.md rename to nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/04_op_template_basics.md b/nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/04_op_template_basics.md rename to nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/05_op_template_payloads.md b/nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/05_op_template_payloads.md rename to nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/06_op_template_variations.md b/nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/06_op_template_variations.md rename to nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/07_template_variables.md b/nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/07_template_variables.md rename to nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md diff --git a/nb-apis/adapters-api/src/main/resources/workload_definition/08_parsed_op_api.md b/nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md similarity index 100% rename from nb-apis/adapters-api/src/main/resources/workload_definition/08_parsed_op_api.md rename to nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/PromPushReporterComponent.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/PromPushReporterComponent.java index c1b822d74..17cb3a053 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/PromPushReporterComponent.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/PromPushReporterComponent.java @@ -112,7 +112,7 @@ public class PromPushReporterComponent extends PeriodicTaskComponent { } PromPushReporterComponent.logger.debug("formatted {} metrics in prom expo format", total); final String exposition = sb.toString(); - logger.trace(() -> "prom exposition format:\n" + exposition); +// logger.trace(() -> "prom exposition format:\n" + exposition); final double backoffRatio = 1.5; final double maxBackoffSeconds = 10; diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/stats/StatBucket.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/stats/StatBucket.java index 7942542f5..3f58a9000 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/stats/StatBucket.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/stats/StatBucket.java @@ -47,21 +47,21 @@ public final class StatBucket { mean = value; dSquared = 0.0d; } else if (Double.isNaN(popped)) { - var newMean = mean + ((value - mean) / ringbuf.count()); - var dSquaredIncrement = ((value - newMean) * (value - mean)); + double newMean = mean + ((value - mean) / ringbuf.count()); + double dSquaredIncrement = ((value - newMean) * (value - mean)); // If this value is too small to be interpreted as a double it gets converted to // zero, which is not what we want. So we use the smallest possible double value if (dSquaredIncrement == 0) dSquaredIncrement = Double.MIN_VALUE; dSquared += dSquaredIncrement; mean = newMean; } else { - var meanIncrement = (value - popped) / ringbuf.count(); - var newMean = mean + meanIncrement; - var dSquaredIncrement = ((value - popped) * (value - newMean + popped - mean)); + double meanIncrement = (value - popped) / ringbuf.count(); + double newMean = mean + meanIncrement; + double dSquaredIncrement = ((value - popped) * (value - newMean + popped - mean)); // If this value is too small to be interpreted as a double it gets converted to // zero, which is not what we want. So we use the smallest possible double value if (dSquaredIncrement == 0) dSquaredIncrement = Double.MIN_VALUE; - var newDSquared = this.dSquared + dSquaredIncrement; + double newDSquared = this.dSquared + dSquaredIncrement; mean = newMean; dSquared = newDSquared; } @@ -89,7 +89,7 @@ public final class StatBucket { public boolean equals(Object obj) { if (obj == this) return true; if (obj == null || obj.getClass() != this.getClass()) return false; - var that = (StatBucket) obj; + StatBucket that = (StatBucket) obj; return this.ringbuf.count() == that.ringbuf.count() && Double.doubleToLongBits(this.mean) == Double.doubleToLongBits(that.mean); } diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpAssertions.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpAssertions.java new file mode 100644 index 000000000..7db6919c8 --- /dev/null +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpAssertions.java @@ -0,0 +1,62 @@ +package io.nosqlbench.engine.api.activityimpl; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.Validator; +import io.nosqlbench.adapters.api.activityimpl.uniform.ValidatorSource; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.AssertingOpDispenser; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.nb.api.errors.OpConfigError; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; + +/// This is a functional wrapper layer which will upgrade a basic [CycleOp] to +/// one that has a validator provided by it's [DriverAdapter], so long as the +/// adapter implements [ValidatorSource] and can provide one for the given name. +public class OpAssertions { + + public final static Logger logger = LogManager.getLogger(OpAssertions.class); + + public static , SPACE extends Space> OpDispenser wrapOptionally( + DriverAdapter adapter, OpDispenser dispenser, + ParsedOp pop + ) { + + Optional validatorName = pop.takeOptionalStaticValue("validator", String.class); + if (validatorName.isEmpty()) return dispenser; + + if (adapter instanceof ValidatorSource vs) { + Optional validator = vs.getValidator(validatorName.get(),pop); + if (validator.isEmpty()) { + throw new OpConfigError( + "a validator '" + validatorName.get() + "' was requested, but adapter '" + adapter.getAdapterName() + "' did not find it."); + } + return new AssertingOpDispenser(adapter, pop, dispenser, validator.get()); + } else { + throw new OpConfigError( + "a validator '" + validatorName.get() + "' was specified, " + "but the adapter '" + adapter.getAdapterName() + "' does " + "not implement " + adapter.getClass().getSimpleName()); + } + } +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpCapture.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpCapture.java new file mode 100644 index 000000000..47ac798f9 --- /dev/null +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpCapture.java @@ -0,0 +1,67 @@ +package io.nosqlbench.engine.api.activityimpl; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.CapturingOpDispenser; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.nb.api.errors.OpConfigError; +import io.nosqlbench.virtdata.core.templates.CapturePoints; +import io.nosqlbench.virtdata.core.templates.UniformVariableCapture; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.function.Function; + +/// This is a functional wrapper layer which will upgrade a basic [CycleOp] to +/// one that has a specialized variable extractor, given +/// 1. The adapter implements [io.nosqlbench.virtdata.core.templates.UniformVariableCapture] +/// 2. The op template has captures defined. Captures can be defined using the +/// capture points as described in [io.nosqlbench.virtdata.core.templates.CapturePointParser], +/// _or_ they can be directly defined on the op template with the reserved op field `capture`. +public class OpCapture { + + public final static Logger logger = LogManager.getLogger(OpCapture.class); + + public static , SPACE extends Space> OpDispenser wrapOptionally( + DriverAdapter adapter, OpDispenser dispenser, + ParsedOp pop + ) { + + CapturePoints captures = pop.getCaptures(); + if (captures.isEmpty()) { + return dispenser; + } + OP op = dispenser.getOp(0L); + + if (op instanceof UniformVariableCapture captureF) { + Function> function = captureF.initCaptureF(captures); + return new CapturingOpDispenser(adapter, pop, dispenser, function); + } else { + throw new OpConfigError( + "variable capture configuration failed because adapter " + adapter + " does not " + + "implement " + UniformVariableCapture.class.getSimpleName() + ); + } + } +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDryrun.java similarity index 72% rename from nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java rename to nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDryrun.java index 389905153..2b9291c19 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpWrappers.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpDryrun.java @@ -22,27 +22,26 @@ import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.activityimpl.uniform.Space; import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; -import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.DryCycleOpDispenserWrapper; -import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.EmitterCycleOpDispenserWrapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.DryrunOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.ResultPrintingOpDispenser; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class OpWrappers { +public class OpDryrun { - public final static Logger logger = LogManager.getLogger(OpWrappers.class); + public final static Logger logger = LogManager.getLogger(OpDryrun.class); - public static , SPACE extends Space> OpDispenser wrapOptionally( - DriverAdapter adapter, - OpDispenser dispenser, + public static , SPACE extends Space> OpDispenser wrapOptionally( + DriverAdapter adapter, + OpDispenser dispenser, ParsedOp pop, - String dryrunSpec + Dryrun dryrun ) { - Dryrun dryrun = Dryrun.valueOf(dryrunSpec); return switch (dryrun) { case none -> dispenser; - case op -> new DryCycleOpDispenserWrapper(adapter, pop, dispenser); - case emit -> new EmitterCycleOpDispenserWrapper(adapter, pop, dispenser); + case op -> new DryrunOpDispenser(adapter, pop, dispenser); + case emit -> new ResultPrintingOpDispenser(adapter, pop, dispenser); case jsonnet -> dispenser; }; } diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpFunctionComposition.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpFunctionComposition.java new file mode 100644 index 000000000..c69d56d55 --- /dev/null +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/OpFunctionComposition.java @@ -0,0 +1,77 @@ +package io.nosqlbench.engine.api.activityimpl; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.Space; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import io.nosqlbench.adapters.api.activityimpl.uniform.opwrappers.*; +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.nb.api.errors.ResultVerificationError; + +/// This is where operations are customized to support special +/// testing or analysis capabilities. The behaviors here are not adapter-specific, +/// but can be specialized by adapter-provided logic in some cases. +/// +/// In every case, the original dispenser and op instances are retained as in normal cycle +/// execution, although they may be used differently. For each new behavior, both the +/// original dispenser and the original op produced by it are wrapped by custom logic which can +/// functionally override or extend them. +/// +/// The current behaviors which are supported are assembled in a fixed topology when used. In +/// other words, users can't simply layer customizations in any order or arrangement they choose. +/// +/// To illustrate, an example of all of these together will be presented below: +/// +/// ``` +/// dryrun ( verify ( print ( capture ( op )))) +///``` +/// +/// This will change to be more in a future refinement, when all op synthesis modifiers will +/// be subsumed into [BaseOpDispenser] +/// +/// Working from the outside-in, each is explained here: +/// * [DryrunOp] (dryrun) - Do not exeucute the operation, but prepare as if you were going to ( +/// including any modifiers) and continue +/// * [AssertingOp] (verify) - apply any assertions specified by the user, and throw an +/// [ResultVerificationError] if the assertions are false, or +/// continue if assertions are true +/// * [ResultPrintingOp] (print) - Print the result of executing the op and continue +/// * [CapturingOp] (capture) - Capture the specified set of named fields from the result of the +/// operation, and continue +public class OpFunctionComposition { + + public final static Logger logger = LogManager.getLogger(OpFunctionComposition.class); + + public static , SPACE extends Space> OpDispenser wrapOptionally( + DriverAdapter adapter, OpDispenser dispenser, + ParsedOp pop, Dryrun dryrun + ) { + + dispenser = OpCapture.wrapOptionally(adapter, dispenser, pop); + dispenser = OpAssertions.wrapOptionally(adapter, dispenser, pop); + dispenser = OpDryrun.wrapOptionally(adapter, dispenser, pop, dryrun); + + return dispenser; + } +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/ResultValidatingDispenser.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/ResultValidatingDispenser.java new file mode 100644 index 000000000..035aa027d --- /dev/null +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/ResultValidatingDispenser.java @@ -0,0 +1,22 @@ +package io.nosqlbench.engine.api.activityimpl; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public class ResultValidatingDispenser { +} diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 61cba931e..e04c5cf40 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -426,9 +426,10 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok DriverAdapter, Space> adapter = adapters.get(i); OpMapper, Space> opMapper = adapter.getOpMapper(); LongFunction spaceFunc = adapter.getSpaceFunc(pop); - OpDispenser> dispenser = opMapper.apply(this, pop, spaceFunc); + OpDispenser> dispenser = opMapper.apply(this, pop, spaceFunc); String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none"); - dispenser = OpWrappers.wrapOptionally(adapter, dispenser, pop, dryrunSpec); + Dryrun dryrun = pop.takeEnumFromFieldOr(Dryrun.class, Dryrun.none, "dryrun"); + dispenser = OpFunctionComposition.wrapOptionally(adapter, dispenser, pop, dryrun); // if (strict) { // optemplate.assertConsumed(); @@ -541,7 +542,7 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok @Deprecated(forRemoval = true) protected OpSequence> createOpSequence(Function> opinit, boolean strict, DriverAdapter defaultAdapter) { - var stmts = loadOpTemplates(defaultAdapter); + List stmts = loadOpTemplates(defaultAdapter); List ratios = new ArrayList<>(stmts.size()); diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java index 07d80f9c3..b30532fcf 100644 --- a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java @@ -28,87 +28,89 @@ import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.nb.api.errors.OpConfigError; import io.nosqlbench.virtdata.core.bindings.DataMapper; import io.nosqlbench.virtdata.core.bindings.VirtData; -import io.nosqlbench.virtdata.core.templates.BindPoint; -import io.nosqlbench.virtdata.core.templates.CapturePoint; -import io.nosqlbench.virtdata.core.templates.ParsedTemplateString; -import io.nosqlbench.virtdata.core.templates.StringBindings; +import io.nosqlbench.virtdata.core.templates.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.*; import java.util.function.Function; import java.util.function.LongFunction; -import java.util.function.LongToIntFunction; /** - * A parsed map template, which allows construction of extracted or projected functions related - * to dynamic value templates. This is the backing implementation for ParsedOp which is map-based - * at the root level, but which can have recursive structure and thus needed an inner value type. - * This implementation supports parsing/restructuring around string, map, and list templates. - * - * The provided map is interpreted as a map of string to object templates using these rules: - *
        - *
      1. If the value is a String and contains no binding points, it is interpreted as a literal
        2. - *
      2. If the value is a String and contains only a binding point with no leading nor trailing text, it is interpreted as an object binding
        3. - *
      3. If the value is a String and contains a binding point with any leading or trailing text, it is interpreted as a String template binding
        4. - *
      4. If the value is a map, list, or set, then each element is interpreted as above
        5. - *
      - * - * TODO: Proactively check casting on functional methods, fallback to {@link NBTypeConverter} only if needed and can - */ + A parsed map template, which allows construction of extracted or projected functions related + to dynamic value templates. This is the backing implementation for ParsedOp which is map-based + at the root level, but which can have recursive structure and thus needed an inner value type. + This implementation supports parsing/restructuring around string, map, and list templates. + + The provided map is interpreted as a map of string to object templates using these rules: +
        +
      1. If the value is a String and contains no binding points, it is interpreted as a literal
        2. +
      2. If the value is a String and contains only a binding point with no leading nor trailing text, + it + is interpreted as an object binding
        3. +
      3. If the value is a String and contains a binding point with any leading or trailing text, it is + interpreted as a String template binding
        4. +
      4. If the value is a map, list, or set, then each element is interpreted as above
        5. +
      + + TODO: Proactively check casting on functional methods, fallback to {@link NBTypeConverter} only if needed and can */ public class ParsedTemplateMap implements LongFunction>, StaticFieldReader, DynamicFieldReader { private final static Logger logger = LogManager.getLogger(ParsedTemplateMap.class); /** - * The fields which are statically assigned + The fields which are statically assigned **/ private final Map statics = new LinkedHashMap<>(); /** - * The fields which are dynamic, and must be realized via functions. - * This map contains keys which identify the field names, and values, which may be null or undefined. + The fields which are dynamic, and must be realized via functions. + This map contains keys which identify the field names, and values, which may be null or + undefined. */ private final Map> dynamics = new LinkedHashMap<>(); /** - * The names of payload values in the result of the operation which should be saved. - * The keys in this map represent the name of the value as it would be found in the native - * representation of a result. If the values are defined, then each one represents the name - * that the found value should be saved as instead of the original name. + The names of payload values in the result of the operation which should be saved. + The keys in this map represent the name of the value as it would be found in the native + representation of a result. If the values are defined, then each one represents the name + that the found value should be saved as instead of the original name. */ - private final List captures = new ArrayList<>(); + private final CapturePoints captures = new CapturePoints(); private final int mapsize; /** - * A prototype of the fully generated map, to be used as the starting point - * when rendering the full map with dynamic values. + A prototype of the fully generated map, to be used as the starting point + when rendering the full map with dynamic values. */ private final LinkedHashMap protomap = new LinkedHashMap<>(); /** - * Any auxiliary source of values to be applied beyond what is specified directly in the op fields. - * This includes, for example, the activity parameters which are allowed by the config model on - * an adapter. This means that you can specify defaults for an op field outside of the workload/op - * templates simply by providing them on the command line or activity parameters otherwise. - * This is exactly how the required op field `driver` works. + Any auxiliary source of values to be applied beyond what is specified directly in the op fields. + This includes, for example, the activity parameters which are allowed by the config model on + an adapter. This means that you can specify defaults for an op field outside of the workload/op + templates simply by providing them on the command line or activity parameters otherwise. + This is exactly how the required op field `driver` works. */ private final List> cfgsources; /** - * This remembers the original template object so that diagnostic and debugging views - * may see the original specifiers, whether they are literals of any type, or a string - * value which is recognized as being or containing some dynamic span, i.e. bind points. + This remembers the original template object so that diagnostic and debugging views + may see the original specifiers, whether they are literals of any type, or a string + value which is recognized as being or containing some dynamic span, i.e. bind points. */ private Map originalTemplateObject; /** - * The bindings definitions from the raw op template data structure. + The bindings definitions from the raw op template data structure. */ private Map bindings; private final String name; - public ParsedTemplateMap(String name, Map map, Map bindings, List> cfgsources) { + public ParsedTemplateMap( + String name, Map map, Map bindings, + List> cfgsources + ) { this.name = name; this.cfgsources = cfgsources; applyTemplateFields(map, bindings); @@ -152,7 +154,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } }); Map submap = (Map) v; - ParsedTemplateMap subtpl = new ParsedTemplateMap(getName(), submap, bindings, cfgsources); + ParsedTemplateMap subtpl = new ParsedTemplateMap( + getName(), submap, bindings, cfgsources); this.captures.addAll(subtpl.getCaptures()); if (subtpl.isStatic()) { statics.put(k, submap); @@ -183,12 +186,12 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } - public List getCaptures() { + public CapturePoints getCaptures() { return this.captures; } /** - * @return true if any field of this template map is dynamic + @return true if any field of this template map is dynamic */ public boolean isDynamic() { return (dynamics.size() > 0); @@ -222,7 +225,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi if (!cfgs.containsKey(key)) { cfgs.put(key, cfgsource.get(key)); } else { - logger.warn("config sources contain overlapping keys for '" + key + "', precedence is undefined"); + logger.warn( + "config sources contain overlapping keys for '" + key + "', precedence is undefined"); } } } @@ -230,11 +234,12 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * create a map of op field names and values, containing all - * statically and dynamically defined fields, but not including - * auxilliary config like params or activity params. - * @param value The input value to the binding functions - * @return A {@link Map} of {@link String} to {@link Object} + create a map of op field names and values, containing all + statically and dynamically defined fields, but not including + auxilliary config like params or activity params. + @param value + The input value to the binding functions + @return A {@link Map} of {@link String} to {@link Object} */ @Override public Map apply(long value) { @@ -252,7 +257,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * @return true if the specified field is found in the dynamic op fields + @return true if the specified field is found in the dynamic op fields */ @Override public boolean isDynamic(String field) { @@ -260,22 +265,25 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * @return true if and only if the named field is present in the static field map. + @return true if and only if the named field is present in the static field map. */ public boolean isStatic(String field) { return statics.containsKey(field); } /** - * @return true if and only if the named field is present in the static field map and the type is assignable to the specified class + @return true if and only if the named field is present in the static field map and the type is + assignable to the specified class */ public boolean isStatic(String field, Class type) { return statics.containsKey(field) && type.isAssignableFrom(field.getClass()); } /** - * @param fields Names of fields to look for in the static, dynamic, or config field maps. - * @return true if and only if all provided field names are present in the static or dynamic or config field maps. + @param fields + Names of fields to look for in the static, dynamic, or config field maps. + @return true if and only if all provided field names are present in the static or dynamic or + config field maps. */ @Override public boolean isDefined(String... fields) { @@ -288,13 +296,17 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * Get the static value for the provided name, cast to the required type. - * - * @param field Name of the field to get - * @param classOfT The type of the field to return. If actual type is not compatible to a cast to this type, then a - * casting error will be thrown. - * @param The parameter type of the return type, used at compile time only to qualify asserted return type - * @return A value of type T, or null + Get the static value for the provided name, cast to the required type. + @param field + Name of the field to get + @param classOfT + The type of the field to return. If actual type is not compatible to a cast to this type, + then a + casting error will be thrown. + @param + The parameter type of the return type, used at compile time only to qualify asserted return + type + @return A value of type T, or null */ @Override public T getStaticValue(String field, Class classOfT) { @@ -328,12 +340,14 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * Get the static value for the provided name, cast to the required type, where the type is inferred - * from the calling context. - * - * @param field Name of the field to get - * @param The parameter type of the return type. used at compile time only to quality return type. - * @return A value of type T, or null + Get the static value for the provided name, cast to the required type, where the type is + inferred + from the calling context. + @param field + Name of the field to get + @param + The parameter type of the return type. used at compile time only to quality return type. + @return A value of type T, or null */ @SuppressWarnings("unchecked") @Override @@ -342,14 +356,17 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * Get the named static field value, or return the provided default, but throw an exception if - * the named field is dynamic. - * - * @param name The name of the field value to return. - * @param defaultValue A value to return if the named value is not present in static nor dynamic fields. - * @param The type of the field to return. - * @return The value - * @throws RuntimeException if the field name is only present in the dynamic fields. + Get the named static field value, or return the provided default, but throw an exception if + the named field is dynamic. + @param name + The name of the field value to return. + @param defaultValue + A value to return if the named value is not present in static nor dynamic fields. + @param + The type of the field to return. + @return The value + @throws RuntimeException + if the field name is only present in the dynamic fields. */ @SuppressWarnings("unchecked") @Override @@ -357,38 +374,46 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi if (isStatic(name)) { return (T) statics.get(name); } else if (dynamics.containsKey(name)) { - throw new BasicError("static field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + - "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); + throw new BasicError( + "static field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); } else { return defaultValue; } } /** - * Get the specified parameter by the user using the defined field which is closest to the op - * template. This is the standard way of getting parameter values which can be specified at the - * op template, op param, or activity level. - * - * @param name The name of the configuration param - * @param defaultValue the default value to return if the value is not defined anywhere in - * (op fields, op params, activity params) - * @param The type of the value to return - * @return A configuration value - * @throws NBConfigError if the named field is defined dynamically, - * as in this case, it is presumed that the parameter is not supported unless it is defined statically. + Get the specified parameter by the user using the defined field which is closest to the op + template. This is the standard way of getting parameter values which can be specified at the + op template, op param, or activity level. + @param name + The name of the configuration param + @param defaultValue + the default value to return if the value is not defined anywhere in + (op fields, op params, activity params) + @param + The type of the value to return + @return A configuration value + @throws NBConfigError + if the named field is defined dynamically, + as in this case, it is presumed that the parameter is not supported unless it is defined + statically. */ public T getStaticConfigOr(String name, T defaultValue) { if (statics.containsKey(name)) { return NBTypeConverter.convertOr(statics.get(name), defaultValue); } + return getConfigValue(name, defaultValue); + } + + private T getConfigValue(String name, T defaultValue) { for (Map cfgsource : cfgsources) { if (cfgsource.containsKey(name)) { return NBTypeConverter.convertOr(cfgsource.get(name), defaultValue); } } if (dynamics.containsKey(name)) { - throw new OpConfigError("static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + - "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); + throw new OpConfigError( + "static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); } else { return defaultValue; } @@ -404,10 +429,11 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } } if (dynamics.containsKey(name)) { - throw new OpConfigError("static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + - "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); + throw new OpConfigError( + "static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); } - throw new OpConfigError("static config field '" + name + "' was requested, but it does not exist"); + throw new OpConfigError( + "static config field '" + name + "' was requested, but it does not exist"); } public T takeStaticConfigOr(String name, T defaultValue) { @@ -416,17 +442,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi protomap.remove(name); return NBTypeConverter.convertOr(value, defaultValue); } - for (Map cfgsource : cfgsources) { - if (cfgsource.containsKey(name)) { - return NBTypeConverter.convertOr(cfgsource.get(name), defaultValue); - } - } - if (dynamics.containsKey(name)) { - throw new OpConfigError("static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + - "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); - } else { - return defaultValue; - } + return getConfigValue(name, defaultValue); } @@ -443,8 +459,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } } if (dynamics.containsKey("name")) { - throw new OpConfigError("static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + - "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); + throw new OpConfigError( + "static config field '" + name + "' was defined dynamically. This may be supportable if the driver developer" + "updates the op mapper to support this field as a dynamic field, but it is not yet supported."); } else { return Optional.empty(); } @@ -452,9 +468,9 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi /** - * Works exactly like {@link #getStaticConfigOr(String, Object)}, except that dynamic values - * at the op field level will be generated on a per-input basis. This is a shortcut method for - * allowing configuration values to be accessed dynamically where it makes sense. + Works exactly like {@link #getStaticConfigOr(String, Object)}, except that dynamic values + at the op field level will be generated on a per-input basis. This is a shortcut method for + allowing configuration values to be accessed dynamically where it makes sense. */ public T getConfigOr(String name, T defaultValue, long input) { if (statics.containsKey(name)) { @@ -473,13 +489,16 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi /** - * Return an optional value for the named field. This is an {@link Optional} form of {@link #getStaticValue}. - * - * @param field Name of the field to get - * @param classOfT The type of field to return. If the actual type is not compatible to a cast to this type, - * then a casting error will be thrown. - * @param The parameter type of the return - * @return An optional value, empty unless the named value is defined in the static field map. + Return an optional value for the named field. This is an {@link Optional} form of + {@link #getStaticValue}. + @param field + Name of the field to get + @param classOfT + The type of field to return. If the actual type is not compatible to a cast to this type, + then a casting error will be thrown. + @param + The parameter type of the return + @return An optional value, empty unless the named value is defined in the static field map. */ @Override public Optional getOptionalStaticValue(String field, Class classOfT) { @@ -487,15 +506,19 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * Get the named field value for a given long input. This uses parameter type inference -- The casting - * to the return type will be based on the type of any assignment or casting on the caller's side. - * Thus, if the actual type is not compatable to a cast to the needed return type, a casting error will - * be thrown. - * - * @param field The name of the field to get. - * @param input The seed value, or cycle value for which to generate the value. - * @param The parameter type of the returned value. Inferred from usage context. - * @return The value. + Get the named field value for a given long input. This uses parameter type inference -- The + casting + to the return type will be based on the type of any assignment or casting on the caller's side. + Thus, if the actual type is not compatable to a cast to the needed return type, a casting error + will + be thrown. + @param field + The name of the field to get. + @param input + The seed value, or cycle value for which to generate the value. + @param + The parameter type of the returned value. Inferred from usage context. + @return The value. */ @SuppressWarnings("unchecked") @Override @@ -513,8 +536,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * @return a set of names which are defined, whether in static fields or dynamic fields, - * but NOT including params nor other config + @return a set of names which are defined, whether in static fields or dynamic fields, + but NOT including params nor other config */ public Set getOpFieldNames() { HashSet nameSet = new HashSet<>(statics.keySet()); @@ -523,30 +546,33 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * Get the op field as a {@link LongFunction} of String. This is a convenience form for - * {@link #getAsRequiredFunction(String, Class)} - * - * @param name The field name which must be defined as static or dynamic - * @return A function which can provide the named field value + Get the op field as a {@link LongFunction} of String. This is a convenience form for + {@link #getAsRequiredFunction(String, Class)} + @param name + The field name which must be defined as static or dynamic + @return A function which can provide the named field value */ public LongFunction getAsRequiredFunction(String name) { return getAsRequiredFunction(name, String.class); } - public > Optional> getAsOptionalEnumFunction(String name, Class type) { + public > Optional> getAsOptionalEnumFunction( + String name, Class type) { Optional> nameFunc = this.getAsOptionalFunction(name, String.class); return nameFunc.map((f) -> (l) -> Enum.valueOf((Class) type, f.apply(l))); } /** - * Get the op field as a {@link LongFunction}, using statics, then dynamics, then config sources. - * - * @param name The field name which must be defined as static or dynamic - * @param type The value type which the field must be assignable to - * @return A function which can provide a value for the given name and type + Get the op field as a {@link LongFunction}, using statics, then dynamics, then config sources. + @param name + The field name which must be defined as static or dynamic + @param type + The value type which the field must be assignable to + @return A function which can provide a value for the given name and type */ @SuppressWarnings("unchecked") - public Optional> getAsOptionalFunction(String name, Class type) { + public Optional> getAsOptionalFunction( + String name, Class type) { if (isStatic(name)) { V value = getStaticValue(name); if (type.isAssignableFrom(value.getClass())) { @@ -556,9 +582,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi return Optional.of((cycle) -> converted); } else { throw new OpConfigError( - "function for '" + name + "' yielded a " + value.getClass().getCanonicalName() - + " type, which is not assignable to a '" + type.getCanonicalName() + "'" - ); + "function for '" + name + "' yielded a " + value.getClass().getCanonicalName() + " type, which is not assignable to a '" + type.getCanonicalName() + "'"); } } else if (isDynamic(name)) { Object testValue = dynamics.get(name).apply(0L); @@ -568,8 +592,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi return Optional.of(l -> NBTypeConverter.convert(dynamics.get(name).apply(l), type)); } else { throw new OpConfigError( - "function for '" + name + "' yielded a " + testValue.getClass().getCanonicalName() - + " type, which is not assignable to '" + type.getCanonicalName() + "'"); + "function for '" + name + "' yielded a " + testValue.getClass().getCanonicalName() + " type, which is not assignable to '" + type.getCanonicalName() + "'"); } } else { if (isConfig(name)) { @@ -580,8 +603,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi return Optional.of(l -> NBTypeConverter.convert(cfgval, type)); } else { throw new OpConfigError( - "function for '" + name + "' found a " + cfgval.getClass().getCanonicalName() - + " type in cfg source, which is not assignable to '" + type.getCanonicalName() + "'"); + "function for '" + name + "' found a " + cfgval.getClass().getCanonicalName() + " type in cfg source, which is not assignable to '" + type.getCanonicalName() + "'"); } } @@ -591,19 +613,22 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi public LongFunction getAsRequiredFunction(String name, Class type) { Optional> sf = getAsOptionalFunction(name, type); - return sf.orElseThrow(() -> new OpConfigError("The op field '" + name + "' is required, but it wasn't found " + - "in the op template named '" + this.getName() + "'")); + return sf.orElseThrow(() -> new OpConfigError( + "The op field '" + name + "' is required, but it wasn't found " + "in the op template named '" + this.getName() + "'")); } /** - * Get a LongFunction which returns either the static value, the dynamic value, or the default value, - * in that order, depending on where it is found first. - * - * @param name The param name for the value - * @param defaultValue The default value to provide the value is not defined for static nor dynamic - * @param The type of value to return - * @return A {@link LongFunction} of type V + Get a LongFunction which returns either the static value, the dynamic value, or the default + value, + in that order, depending on where it is found first. + @param name + The param name for the value + @param defaultValue + The default value to provide the value is not defined for static nor dynamic + @param + The type of value to return + @return A {@link LongFunction} of type V */ @Override public LongFunction getAsFunctionOr(String name, V defaultValue) { @@ -621,19 +646,28 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi /** - * Get a LongFunction that first creates a LongFunction of String as in {@link #getAsFunctionOr(String, Object)} )}, but then - * applies the result and cached it for subsequent access. This relies on {@link ObjectCache} internally. - * The purpose of this is to avoid costly re-computation for mapped values over pure functions where the computation - * cost is significantly high. For trivial functions, the cost is generally lower than the hash lookup within the - * object cache.. - * - * @param fieldname The name of the field which could contain a static or dynamic value - * @param defaultValue The default value to use in the init function if the fieldname is not defined as static nor dynamic - * @param init A function to apply to the value to produce the product type - * @param The type of object to return - * @return A caching function which chains to the init function, with caching + Get a LongFunction that first creates a LongFunction of String as in + {@link #getAsFunctionOr(String, Object)} )}, but then + applies the result and cached it for subsequent access. This relies on {@link ObjectCache} + internally. + The purpose of this is to avoid costly re-computation for mapped values over pure functions + where the computation + cost is significantly high. For trivial functions, the cost is generally lower than the hash + lookup within the + object cache.. + @param fieldname + The name of the field which could contain a static or dynamic value + @param defaultValue + The default value to use in the init function if the fieldname is not defined as static nor + dynamic + @param init + A function to apply to the value to produce the product type + @param + The type of object to return + @return A caching function which chains to the init function, with caching */ - public LongFunction getAsCachedFunctionOr(String fieldname, String defaultValue, Function init) { + public LongFunction getAsCachedFunctionOr( + String fieldname, String defaultValue, Function init) { // caching is only valid for the dynamic case, everything else can elide it if (isDynamic(fieldname)) { LongFunction f = l -> get(fieldname, l); @@ -644,10 +678,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi return l -> oc.apply(fs.apply(l)); } else { throw new OpConfigError( - "Unable to compose string func to obj cache with non-String function of type " - + f.getClass().getCanonicalName() + " in op template named '" - + this.getName() + "'" - ); + "Unable to compose string func to obj cache with non-String function of type " + f.getClass().getCanonicalName() + " in op template named '" + this.getName() + "'"); } } if (isStatic(fieldname)) { @@ -656,9 +687,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi V defaultObject = init.apply(s); return l -> defaultObject; } else { - throw new OpConfigError("Unable to compose string to object cache with non-String value " + - "of type " + defaultValue.getClass().getCanonicalName() - + " in op template named '" + this.getName() + "'"); + throw new OpConfigError( + "Unable to compose string to object cache with non-String value " + "of type " + defaultValue.getClass().getCanonicalName() + " in op template named '" + this.getName() + "'"); } } if (isConfig(fieldname)) { @@ -667,7 +697,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi V defaultObject = init.apply(s); return l -> defaultObject; } else { - throw new OpConfigError("Unable to compose string to object cache with non-String value of type " + defaultValue.getClass().getCanonicalName()); + throw new OpConfigError( + "Unable to compose string to object cache with non-String value of type " + defaultValue.getClass().getCanonicalName()); } } else { V defaultObject = init.apply(defaultValue); @@ -677,27 +708,31 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi /** - * @param field The requested field name - * @return true if the named field is defined as static or dynamic or config (params and activity params) + @param field + The requested field name + @return true if the named field is defined as static or dynamic or config (params and activity params) */ public boolean isDefined(String field) { return statics.containsKey(field) || dynamics.containsKey(field) || isConfig(field); } /** - * Inverse of {@link #isDefined(String)}, provided for clarify in some situations - * - * @param field The field name - * @return true if the named field is defined neither as static nor as dynamic + Inverse of {@link #isDefined(String)}, provided for clarify in some situations + @param field + The field name + @return true if the named field is defined neither as static nor as dynamic */ public boolean isUndefined(String field) { return !(statics.containsKey(field) || dynamics.containsKey(field) || isConfig(field)); } /** - * @param field The requested field name - * @param type The required type of the field value - * @return true if the named field is defined as static or dynamic and the value produced can be assigned to the specified type + @param field + The requested field name + @param type + The required type of the field value + @return true if the named field is defined as static or dynamic and the value produced can be + assigned to the specified type */ @Override public boolean isDefined(String field, Class type) { @@ -706,31 +741,24 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi if (type.isAssignableFrom(testObject.getClass())) { return true; } else { - throw new OpConfigError("field " + field + " was defined as a function, " + - "but not one that returns the requested type " - + testObject.getClass().getCanonicalName() - + " in op template named '" + this.getName() + "'" - ); + throw new OpConfigError( + "field " + field + " was defined as a function, " + "but not one that returns the requested type " + testObject.getClass().getCanonicalName() + " in op template named '" + this.getName() + "'"); } } if (isStatic(field)) { if (type.isAssignableFrom(statics.get(field).getClass())) { return true; } else { - throw new OpConfigError("field " + field + " was defined (static), " + - "but not as the requested type " + type.getCanonicalName() - + " in op template named '" + this.getName() + "'" - ); + throw new OpConfigError( + "field " + field + " was defined (static), " + "but not as the requested type " + type.getCanonicalName() + " in op template named '" + this.getName() + "'"); } } if (isConfig(field)) { if (type.isAssignableFrom(getConfig(field).getClass())) { return true; } else { - throw new OpConfigError("field " + field + " was defined (config), " + - "but not as the requested type " + type.getCanonicalName() - + " in op template named '" + this.getName() + "'" - ); + throw new OpConfigError( + "field " + field + " was defined (config), " + "but not as the requested type " + type.getCanonicalName() + " in op template named '" + this.getName() + "'"); } } return false; @@ -760,12 +788,13 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * Take the value of the specified field from the dynamic or static layers, or reference it - * from the config layer without removal. Then, flatten any string, list, or map structures - * into a map of strings with names injected as needed. Then, convert the values to string - * templates and return that. - * @param fieldname the field to take the templates from - * @return A map of templates, or an empty map if the field is not defined or is empty. + Take the value of the specified field from the dynamic or static layers, or reference it + from the config layer without removal. Then, flatten any string, list, or map structures + into a map of strings with names injected as needed. Then, convert the values to string + templates and return that. + @param fieldname + the field to take the templates from + @return A map of templates, or an empty map if the field is not defined or is empty. */ public Map takeAsNamedTemplates(String fieldname) { Object entry = originalTemplateObject.get(fieldname); @@ -800,11 +829,11 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi elements.put(this.getName() + "-verifier-" + k, v); }); } - Map parsedStringTemplates - = new LinkedHashMap<>(); + Map parsedStringTemplates = new LinkedHashMap<>(); elements.forEach((k, v) -> { if (v instanceof CharSequence chars) { - parsedStringTemplates.put(k, new ParsedTemplateString(chars.toString(), this.bindings)); + parsedStringTemplates.put( + k, new ParsedTemplateString(chars.toString(), this.bindings)); } }); return parsedStringTemplates; @@ -817,15 +846,16 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi if (fval instanceof CharSequence) { return Optional.of(new ParsedTemplateString(fval.toString(), this.bindings)); } else { - throw new RuntimeException("Can not make a parsed text template from op template field '" + fieldname + "' of type '" + fval.getClass().getSimpleName() + "'"); + throw new RuntimeException( + "Can not make a parsed text template from op template field '" + fieldname + "' of type '" + fval.getClass().getSimpleName() + "'"); } } return Optional.empty(); } /** - * convenience method for conjugating {@link #isDefined(String)} with AND - * @return true if all specified fields are defined as static or dynamic or config + convenience method for conjugating {@link #isDefined(String)} with AND + @return true if all specified fields are defined as static or dynamic or config */ public boolean isDefinedAll(String... fields) { for (String field : fields) { @@ -837,58 +867,65 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * @param fields The ordered field names for which the {@link ListBinder} will be created - * @return a new {@link ListBinder} which can produce a {@link List} of Objects from a long input. + @param fields + The ordered field names for which the {@link ListBinder} will be created + @return a new {@link ListBinder} which can produce a {@link List} of Objects from a long input. */ public LongFunction> newListBinder(String... fields) { return new ListBinder(this, fields); } /** - * @param fields The ordered field names for which the {@link ListBinder} will be created - * @return a new {@link ListBinder} which can produce a {@link List} of Objects from a long input. + @param fields + The ordered field names for which the {@link ListBinder} will be created + @return a new {@link ListBinder} which can produce a {@link List} of Objects from a long input. */ public LongFunction> newListBinder(List fields) { return new ListBinder(this, fields); } /** - * @param fields The ordered field names for which the {@link OrderedMapBinder} will be created - * @return a new {@link OrderedMapBinder} which can produce a {@link Map} of String to Objects from a long input. + @param fields + The ordered field names for which the {@link OrderedMapBinder} will be created + @return a new {@link OrderedMapBinder} which can produce a {@link Map} of String to Objects from + a long input. */ public LongFunction> newOrderedMapBinder(String... fields) { return new OrderedMapBinder(this, fields); } /** - * @param fields The ordered field names for which the {@link ArrayBinder} will be created - * @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. + @param fields + The ordered field names for which the {@link ArrayBinder} will be created + @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. */ public LongFunction newArrayBinder(String... fields) { return new ArrayBinder(this, fields); } /** - * @param fields The ordered field names for which the {@link ArrayBinder} will be created - * @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. + @param fields + The ordered field names for which the {@link ArrayBinder} will be created + @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. */ public LongFunction newArrayBinder(List fields) { return new ArrayBinder(this, fields); } /** - * @param bindPoints The {@link BindPoint}s for which the {@link ArrayBinder} will be created - * @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. + @param bindPoints + The {@link BindPoint}s for which the {@link ArrayBinder} will be created + @return a new {@link ArrayBinder} which can produce a {@link Object} array from a long input. */ public LongFunction newArrayBinderFromBindPoints(List bindPoints) { return new ArrayBinder(bindPoints); } /** - * Get the {@link LongFunction} which is used to resolve a dynamic field value. - * - * @param field The field name for a dynamic parameter - * @return The mapping function + Get the {@link LongFunction} which is used to resolve a dynamic field value. + @param field + The field name for a dynamic parameter + @return The mapping function */ public LongFunction getMapper(String field) { LongFunction mapper = dynamics.get(field); @@ -896,7 +933,7 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi } /** - * @return the logical map size, including all static and dynamic fields + @return the logical map size, including all static and dynamic fields */ public int getSize() { return this.mapsize; @@ -913,16 +950,12 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi if (isConfig(fieldname)) { return getConfig(fieldname).getClass(); } - throw new OpConfigError("Unable to determine value type for undefined op field '" - + fieldname + "'" + " in op template named '" + this.getName() + "'"); + throw new OpConfigError( + "Unable to determine value type for undefined op field '" + fieldname + "'" + " in op template named '" + this.getName() + "'"); } public , V> Optional> getOptionalTargetEnum( - Class enumclass, - String typeFieldName, - String valueFieldName, - Class valueClass - ) { + Class enumclass, String typeFieldName, String valueFieldName, Class valueClass) { if (isStatic(typeFieldName)) { String enumValue = statics.get(typeFieldName).toString(); E verifiedEnumValue; @@ -930,72 +963,75 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi try { verifiedEnumValue = Enum.valueOf(enumclass, enumValue); } catch (IllegalArgumentException iae) { - throw new OpConfigError("type designator field '" + typeFieldName - + "' had value of '" + enumValue + ", but this failed to match " + - "any of known types in " + EnumSet.allOf(enumclass) - + " in op template named '" + this.getName() + "'"); + throw new OpConfigError( + "type designator field '" + typeFieldName + "' had value of '" + enumValue + ", but this failed to match " + "any of known types in " + EnumSet.allOf( + enumclass) + " in op template named '" + this.getName() + "'"); } if (isDefined(valueFieldName)) { if (isStatic(typeFieldName)) { return Optional.of( - new TypeAndTarget(verifiedEnumValue, typeFieldName, l -> NBTypeConverter.convert(statics.get(valueFieldName), valueClass)) - ); + new TypeAndTarget( + verifiedEnumValue, typeFieldName, + l -> NBTypeConverter.convert(statics.get(valueFieldName), valueClass) + )); } else if (isDynamic(valueFieldName)) { return Optional.of( - new TypeAndTarget(verifiedEnumValue, typeFieldName, getAsRequiredFunction(valueFieldName, valueClass)) - ); + new TypeAndTarget( + verifiedEnumValue, typeFieldName, + getAsRequiredFunction(valueFieldName, valueClass) + )); } } } else if (isDynamic(typeFieldName)) { - throw new OpConfigError("The op template field '" + typeFieldName + - "' must be a static value. You can not vary it by cycle" - + " in op template named '" + this.getName() + "'" - ); + throw new OpConfigError( + "The op template field '" + typeFieldName + "' must be a static value. You can not vary it by cycle" + " in op template named '" + this.getName() + "'"); } return Optional.empty(); } public , V> Optional> getOptionalTargetEnum( - Class enumclass, - Class valueClass, - String alternateTypeField, + Class enumclass, Class valueClass, String alternateTypeField, String alternateValueField ) { Optional> result = getOptionalTargetEnum(enumclass, valueClass); if (result.isPresent()) { return result; } - return getOptionalTargetEnum(enumclass, alternateTypeField, alternateValueField, valueClass); + return getOptionalTargetEnum( + enumclass, alternateTypeField, alternateValueField, valueClass); } /** - * Given an enum of any type, return the enum value which is found in any of the field names of the op template, - * ignoring case and any non-word characters. This is useful for matching op templates to op types where the presence - * of a field determines the type. Further, if there are multiple matching names, an {@link OpConfigError} is thrown to - * avoid possible ambiguity. - * - * @param enumclass The enum class for matching values - * @param Generic type for the enum class - * @return Optionally, an enum value which matches, or {@link Optional#empty()} - * @throws OpConfigError if more than one field matches + Given an enum of any type, return the enum value which is found in any of the field names of the + op template, + ignoring case and any non-word characters. This is useful for matching op templates to op types + where the presence + of a field determines the type. Further, if there are multiple matching names, an + {@link OpConfigError} is thrown to + avoid possible ambiguity. + @param enumclass + The enum class for matching values + @param + Generic type for the enum class + @return Optionally, an enum value which matches, or {@link Optional#empty()} + @throws OpConfigError + if more than one field matches */ public , V> Optional> getOptionalTargetEnum( - Class enumclass, - Class valueClass - ) { + Class enumclass, Class valueClass) { List> matched = new ArrayList<>(); for (E e : EnumSet.allOf(enumclass)) { - String lowerenum = e.name().toLowerCase(Locale.ROOT).replaceAll("[^\\w]", ""); + String lowerenum = e.name().toLowerCase(Locale.ROOT).replaceAll("\\W", ""); for (String s : statics.keySet()) { - String lowerkey = s.toLowerCase(Locale.ROOT).replaceAll("[^\\w]", ""); + String lowerkey = s.toLowerCase(Locale.ROOT).replaceAll("\\W", ""); if (lowerkey.equals(lowerenum)) { matched.add(new TypeAndTarget<>(e, s, null)); } } for (String s : dynamics.keySet()) { - String lowerkey = s.toLowerCase(Locale.ROOT).replaceAll("[^\\w]", ""); + String lowerkey = s.toLowerCase(Locale.ROOT).replaceAll("\\W", ""); if (lowerkey.equals(lowerenum)) { matched.add(new TypeAndTarget<>(e, s, null)); } @@ -1004,82 +1040,73 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi if (matched.size() == 1) { TypeAndTarget prototype = matched.get(0); LongFunction asFunction = getAsRequiredFunction(prototype.field, valueClass); - return Optional.of(new TypeAndTarget(prototype.enumId, prototype.field, asFunction)); + return Optional.of( + new TypeAndTarget(prototype.enumId, prototype.field, asFunction)); } else if (matched.size() > 1) { - throw new OpConfigError("Multiple matches were found from op template fieldnames [" - + getOpFieldNames() + "] to possible enums: [" - + EnumSet.allOf(enumclass) + "]" - + " in op template named '" + this.getName() + "'" - ); + throw new OpConfigError( + "Multiple matches were found from op template fieldnames [" + getOpFieldNames() + "] to possible enums: [" + EnumSet.allOf( + enumclass) + "]" + " in op template named '" + this.getName() + "'"); } return Optional.empty(); } public , V> TypeAndTarget getTargetEnum( - Class enumclass, - Class valueClass, - String tname, - String vname - ) { - Optional> optionalMappedEnum = getOptionalTargetEnum(enumclass, valueClass); + Class enumclass, Class valueClass, String tname, String vname) { + Optional> optionalMappedEnum = getOptionalTargetEnum( + enumclass, valueClass); if (optionalMappedEnum.isPresent()) { return optionalMappedEnum.get(); } - Optional> optionalSpecifiedEnum = getOptionalTargetEnum(enumclass, tname, vname, valueClass); + Optional> optionalSpecifiedEnum = getOptionalTargetEnum( + enumclass, tname, vname, valueClass); if (optionalSpecifiedEnum.isPresent()) { return optionalSpecifiedEnum.get(); } - throw new OpConfigError("While mapping op template named '" - + this.getName() + "', Unable to map the type and target for possible values " - + EnumSet.allOf(enumclass) + " either by key or by fields " + tname - + " and " + vname + ". " + - "Fields considered: static:" + statics.keySet() + " dynamic:" - + dynamics.keySet() + " in op template named '" + this.getName() + "'" - ); + throw new OpConfigError( + "While mapping op template named '" + this.getName() + "', Unable to map the type and target for possible values " + EnumSet.allOf( + enumclass) + " either by key or by fields " + tname + " and " + vname + ". " + "Fields considered: static:" + statics.keySet() + " dynamic:" + dynamics.keySet() + " in op template named '" + this.getName() + "'"); } private String getName() { return name; } - public , V> TypeAndTarget getTargetEnum(Class enumclass, Class valueClass) { + public , V> TypeAndTarget getTargetEnum( + Class enumclass, Class valueClass) { Optional> typeFromEnum = getOptionalTargetEnum(enumclass, valueClass); - return typeFromEnum.orElseThrow( - () -> { - String values = EnumSet.allOf(enumclass).toString(); - Set definedNames = getOpFieldNames(); - return new OpConfigError("Unable to match op template fields [" + definedNames - + "] with possible op types [" + values + "] " + - "in op template named '" + this.getName() + "'" + - "If you are specifying an op type which should be implemented, " + - "please file an issue." - ); - } - ); + return typeFromEnum.orElseThrow(() -> { + String values = EnumSet.allOf(enumclass).toString(); + Set definedNames = getOpFieldNames(); + return new OpConfigError( + "Unable to match op template fields [" + definedNames + "] with possible op types [" + values + "] " + "in op template named '" + this.getName() + "'" + "If you are specifying an op type which should be implemented, " + "please file an issue."); + }); } /** - * Map a named op field to an enum - * - * @param enumclass The type of enum to look within - * @param fieldname The field name to look for - * @param The generic type of the enum - * @return An optional enum value + Map a named op field to an enum + @param enumclass + The type of enum to look within + @param fieldname + The field name to look for + @param + The generic type of the enum + @return An optional enum value */ - public > Optional getOptionalEnumFromField(Class enumclass, String fieldname) { + public > Optional getOptionalEnumFromField( + Class enumclass, String fieldname) { Optional enumField = getOptionalStaticConfig(fieldname, String.class); if (enumField.isEmpty()) { return Optional.empty(); } - String lowerv = enumField.get().toLowerCase(Locale.ROOT).replaceAll("[^\\w]", ""); + String lowerv = enumField.get().toLowerCase(Locale.ROOT).replaceAll("\\W", ""); List matched = new ArrayList<>(); for (E e : EnumSet.allOf(enumclass)) { - String lowerenum = e.name().toLowerCase(Locale.ROOT).replaceAll("[^\\w]", ""); + String lowerenum = e.name().toLowerCase(Locale.ROOT).replaceAll("\\W", ""); if (lowerv.equals(lowerenum)) { matched.add(e); } @@ -1088,12 +1115,22 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi return Optional.of(matched.get(0)); } if (matched.size() > 1) { - throw new OpConfigError("Multiple matches were found from op template fieldnames [" - + getOpFieldNames() + "] to possible enums: [" + EnumSet.allOf(enumclass) + "]"); + throw new OpConfigError( + "Multiple matches were found from op template fieldnames [" + getOpFieldNames() + "] to possible enums: [" + EnumSet.allOf( + enumclass) + "]"); } return Optional.empty(); } + public > Optional takeOptionalEnumFromField( + Class enumClass, String fieldName) { + Optional optionalValue = getOptionalEnumFromField(enumClass, fieldName); + if (optionalValue.isPresent()) { + this.statics.remove(fieldName); + } + return optionalValue; + } + public Map parseStaticCmdMap(String taskname, String mainField) { Object mapsrc = getStaticValue(taskname); @@ -1104,7 +1141,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi Object mapsSrc = getStaticValue(key); List> maps = new ArrayList<>(); for (String spec : mapsSrc.toString().split("; +")) { - LinkedHashMap map = new LinkedHashMap<>(ParamsParser.parseToMap(spec, mainField)); + LinkedHashMap map = new LinkedHashMap<>( + ParamsParser.parseToMap(spec, mainField)); maps.add(map); } return maps; @@ -1116,12 +1154,8 @@ public class ParsedTemplateMap implements LongFunction>, StaticFi sb.append("protomap:\n"); for (String k : this.protomap.keySet()) { Object v = this.protomap.get(k); - sb.append(" ") - .append(k) - .append("->") - .append( - v == null ? originalTemplateObject.get(k) : v.toString() - ).append("\n"); + sb.append(" ").append(k).append("->").append( + v == null ? originalTemplateObject.get(k) : v.toString()).append("\n"); } return sb.toString(); diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePoint.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePoint.java index acefc06e9..7af021c16 100644 --- a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePoint.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePoint.java @@ -19,80 +19,119 @@ package io.nosqlbench.virtdata.core.templates; import java.util.Objects; /** - * A capture point is a named variable which should be extracted from a payload or result type - * using a native driver API. The result is meant to be provided to the NoSQLBench runtime - * during cycle execution, and stored in a scoped context of variables which can be re-used within - * other operations. - *
      - *

      Format

      - * - * 
      {@code
- * select [username as u1] from users where userid={userid};
- * }
      - * - * In the example above, the span [username as u1] is recognized as a capture point. - * The name of the variable to be captured is username. It is to be captured under - * a different variable name u1. - * - * If the name is the same in both cases, i.e. the variable is named in the result as it - * should be known after extraction, then you can elide the as u1 clause as in this example: - * - * 
      {@code
- * select [username] from users where userid={userid};
- * }
      - * - * During op mapping, any capture points are condensed down to the native driver vernacular by - * removing the square brackets from the op template. Thus, the result of parsing the above would - * yield a form compatible with a native driver. For example, converting to prepared statement form - * would yield: - * - * 
      {@code
- * select username from users where userid=:userid
- * }
      - * - * For details on the {userid} form, see {@link BindPoint} - */ -public class CapturePoint { + A capture point is a named variable which should be extracted from a payload or result type + using a native driver API. The result is meant to be provided to the NoSQLBench runtime + during cycle execution, and stored in a scoped context of variables which can be re-used within + other operations. +
      +

      Format

      - private final String name; + 
      {@code
+select [username as u1] from users where userid={userid};
+}
      + + In the example above, the span [username as u1] is recognized as a capture point. + The name of the variable to be captured is username. It is to be captured under + a different variable name u1. + + If the name is the same in both cases, i.e. the variable is named in the result as it + should be known after extraction, then you can elide the as u1 clause as in this example: + + 
      {@code
+select [username] from users where userid={userid};
+}
      + + During op mapping, any capture points are condensed down to the native driver vernacular by + removing the square brackets from the op template. Thus, the result of parsing the above would + yield a form compatible with a native driver. For example, converting to prepared statement form + would yield: + + 
      {@code
+select username from users where userid=:userid
+}
      + + For details on the {userid} form, see {@link BindPoint} */ +public class CapturePoint { + + private final String sourceName; private final String asName; + private final Class asCast; + private final String castName; - public CapturePoint(String name, String asName) { - this.name = name; + public CapturePoint(Class asCast, String castName, String anchorName, String asName) { + this.asCast = asCast; + this.castName = castName; + this.sourceName = anchorName; this.asName = asName; } - public String getName() { - return name; + private static Class classForName(String asCast) { + if (asCast == null) return Object.class; + for (String pname : new String[]{"", "java.lang.", "java.util."}) { + try { + String fullName = pname + asCast; + Class clazz = Class.forName(fullName); + if (clazz != null) return clazz; + } catch (ClassNotFoundException ignored) { + } + } + throw new RuntimeException(new ClassNotFoundException(asCast)); + } + + private String nameOfClass(Class cast) { + return cast.getCanonicalName(); + } + + + public String getSourceName() { + return sourceName; + } + + public String getAsName() { + return asName; + } + + public Class getAsCast() { + return asCast; + } + + public static CapturePoint of(String cast, String sourceName, String asName) { + return new CapturePoint(classForName(cast), cast, sourceName, asName != null ? asName : sourceName); + } + + public static CapturePoint of(Class cast, String sourceName, String asName) { + return new CapturePoint(cast, cast.getCanonicalName(), sourceName, asName != null ? asName : sourceName); } /** - * Create a CapturePoint with the specified anchorName, and an optional aliasName. - * If aliasName is null, then the anchorName is used as the alias. - * - * @param anchorName The name of the capture variable in the native form - * @param aliasName The name of the captured value as seen by consumers - * @return A new CapturePoint + Create a CapturePoint with the specified anchorName, and an optional aliasName. + If aliasName is null, then the anchorName is used as the alias. + @param anchorName + The name of the capture variable in the native form + @param asName + The name of the captured value as seen by consumers + @return A new CapturePoint */ - public static CapturePoint of(String anchorName, String aliasName) { + public static CapturePoint of(String anchorName, String asName) { Objects.requireNonNull(anchorName); - return new CapturePoint(anchorName, aliasName == null ? anchorName : aliasName); + return new CapturePoint(null, null, anchorName, asName == null ? anchorName : asName); } /** - * Create a CapturePoint with the specified anchorName, and the same aliasName. - * - * @param anchorName The name of the capture variable in the native form and as seen by consumers. - * @return A new CapturePoint + Create a CapturePoint with the specified anchorName, and the same aliasName. + @param anchorName + The name of the capture variable in the native form and as seen by consumers. + @return A new CapturePoint */ public static CapturePoint of(String anchorName) { Objects.requireNonNull(anchorName); - return new CapturePoint(anchorName, anchorName); + return new CapturePoint(null, null, anchorName, anchorName); } @Override public String toString() { - return "[" + getName() + (name.equals(asName) ? "" : " as " + asName) + "]"; + return "[" + (castName != null ? "(" + castName + ")" : "") + getSourceName() + (sourceName.equals( + asName) ? "" : " as " + asName) + "]"; } @Override @@ -100,11 +139,16 @@ public class CapturePoint { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CapturePoint that = (CapturePoint) o; - return Objects.equals(name, that.name) && Objects.equals(asName, that.asName); + return Objects.equals(sourceName, that.sourceName) && Objects.equals(asName, that.asName); } @Override public int hashCode() { - return Objects.hash(name, asName); + return Objects.hash(sourceName, asName); + } + + public TYPE valueOf(Object v) { + TYPE value = this.asCast.cast(v); + return value; } } diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointException.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointException.java new file mode 100644 index 000000000..e5ac7ec07 --- /dev/null +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointException.java @@ -0,0 +1,25 @@ +package io.nosqlbench.virtdata.core.templates; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public class CapturePointException extends RuntimeException { + public CapturePointException(String message) { + super(message); + } +} diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointParser.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointParser.java index 784707ba0..ce83b915e 100644 --- a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointParser.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointParser.java @@ -22,12 +22,42 @@ import java.util.Objects; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +/// A capture point is named variable which is to be extracted from the +/// result of an operation, using whichever syntax and type conventions +/// that are appropriate to the specific op implementation. +/// The details are discussed in [NB Field Capture +/// Discussion](https://github.com/nosqlbench/nosqlbench/discussions/938) +/// +/// Examples: +/// * capture the value of "field42" (into variable "field42"): `select \[field42],* from ...` +/// * capture the value of "field42" into variable "getitgotitgood": `select \[field42 as +/// getitgotitgood],* from ...` +/// +/// Valid identifiers consist of any word character followed by zero or more word characters, +/// digits, hyphens, +/// underscores, or periods. This applies to both the field name to capture and the variable names +/// to capture +/// their values in. Alternately, the `*` character may be used to indicate that all available +/// fields should be captured. +/// +/// Optional type assertions are allowed which will ensure that the value can be assigned or +/// otherwise coerced +/// into the expected value: +/// * `select \[(List) field42] ...` +/// * `select \[(List)field42)] ...` +/// * `select \[(int[]) field42)] ...` +/// * `select \[(java.lang.Number) field42]` +/// * `select \[(Map) field42] ...` +/// +/// For details on how capture points are used at runtime, consult [[DynamicVariableCapture]] public class CapturePointParser implements Function { public final static Pattern CAPTUREPOINT_PATTERN = Pattern.compile( - "(\\[(?\\w+[-_\\d\\w.]*)(\\s+[aA][sS]\\s+(?\\w+[-_\\d\\w.]*))?])" - ); + "(\\[(\\((?[^)]+)\\))? *(?\\w+[-_\\d\\w.]*|\\*)(\\s+[aA][sS]\\s+" + + "(?\\w+[-_\\d\\w.]*))?])"); + @Override public Result apply(String template) { StringBuilder raw = new StringBuilder(); @@ -35,25 +65,25 @@ public class CapturePointParser implements Function captures = new ArrayList<>(); while (m.find()) { - CapturePoint captured = CapturePoint.of(m.group("capture"), m.group("alias")); + CapturePoint captured = CapturePoint.of( + m.group("cast"), m.group("capture"), m.group("alias")); captures.add(captured); - m.appendReplacement(raw,captured.getName()); + m.appendReplacement(raw, captured.getSourceName()); } m.appendTail(raw); - return new Result(raw.toString(),captures); + return new Result(raw.toString(), new CapturePoints(captures)); } - public final static class Result { + public Result parse(java.util.Map rawdata) { + String specFromMap = rawdata.keySet().stream().map( + ko -> "[" + rawdata.get(ko) + " as " + ko.toString() + "]").collect( + Collectors.joining("")); + return apply(specFromMap); + } - private final String rawTemplate; - private final List captures; - - public Result(String rawTemplate, List captures) { - - this.rawTemplate = rawTemplate; - this.captures = captures; - } + public final static record Result + (String rawTemplate, CapturePoints captures) { public String getRawTemplate() { return this.rawTemplate; @@ -63,14 +93,6 @@ public class CapturePointParser implements Function extends ArrayList> { + + public CapturePoints() {} + + public CapturePoints(@NotNull Collection> capturePoints) { + super(capturePoints); + } + + public boolean isGlob() { + return this.size()==1 && this.get(0).getSourceName().equals("*"); + } +} diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/VariableCapture.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/DynamicVariableCapture.java similarity index 59% rename from nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/VariableCapture.java rename to nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/DynamicVariableCapture.java index 19c3043d2..c962360f4 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/VariableCapture.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/DynamicVariableCapture.java @@ -14,10 +14,9 @@ * limitations under the License. */ -package io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes; - -import io.nosqlbench.virtdata.core.templates.ParsedTemplateString; +package io.nosqlbench.virtdata.core.templates; +import java.util.List; import java.util.Map; /** @@ -29,6 +28,15 @@ import java.util.Map; * and to allow for auto documentation tha the feature is supported for * a given adapter. */ -public interface VariableCapture { - Map capture(); + +/// Any type implementing [[DynamicVariableCapture]] will allow +/// a caller to extract a map of names and values from it. +/// The implementor of [[#capture(List)]] is responsible for extracting the dynamic values, +/// but the caller is expected to use a standard component to do type verification +/// +/// In cases where a type assertion is provided, the values captured dynamically will be subject to an +/// additional phase of type validation and coercion. If required types are not compatible a +/// variable capture error should be thrown. +public interface DynamicVariableCapture { + Map capture(List points, RESULT result); } diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java index 2436ca6e0..23b0e2da0 100644 --- a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java @@ -109,7 +109,7 @@ public class ParsedTemplateString { private final static Logger logger = LogManager.getLogger(ParsedTemplateString.class); private final String rawtemplate; - private final List captures = new ArrayList<>(); + private final CapturePoints captures = new CapturePoints(); private final List bindpoints; public static ParsedTemplateString of(String rawtemplate, Map bindings) { @@ -136,11 +136,10 @@ public class ParsedTemplateString { */ public ParsedTemplateString(String rawtemplate, Map availableBindings) { this.bindings.putAll(availableBindings); - this.rawtemplate = rawtemplate; - CapturePointParser capturePointParser = new CapturePointParser(); CapturePointParser.Result captureData = capturePointParser.apply(rawtemplate); this.captures.addAll(captureData.getCaptures()); + this.rawtemplate = captures.isEmpty() ? rawtemplate : captureData.getRawTemplate(); BindPointParser bindPointParser = new BindPointParser(); BindPointParser.Result bindPointsResult = bindPointParser.apply(captureData.getRawTemplate(), availableBindings); @@ -280,7 +279,7 @@ public class ParsedTemplateString { } } - public List getCaptures() { + public CapturePoints getCaptures() { return this.captures; } diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/UniformVariableCapture.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/UniformVariableCapture.java new file mode 100644 index 000000000..376ec2da4 --- /dev/null +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/UniformVariableCapture.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.core.templates; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * If an op implements VariableCapture, then it is known to be able to + * extract variables from its result. Generally speaking, this should + * be implemented within an Op according to the standard format + * of {@link ParsedTemplateString#getCaptures()}. Any op implementing + * this should use the interface below to support interop between adapters + * and to allow for auto documentation tha the feature is supported for + * a given adapter. + */ + +/// Any type implementing [[UniformVariableCapture]] will allow +/// a caller to extract a map of names and values from it. +/// The implementor of [[#capture(List)]] is responsible for extracting the dynamic values, +/// but the caller is expected to use a standard component to do type verification +/// +/// In cases where a type assertion is provided, the values captured dynamically will be subject to an +/// additional phase of type validation and coercion. If required types are not compatible a +/// variable capture error should be thrown. +public interface UniformVariableCapture { + Function> initCaptureF(CapturePoints points); +} diff --git a/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java b/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java index 7a79d8814..5751e0837 100644 --- a/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java +++ b/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java @@ -19,8 +19,10 @@ package io.nosqlbench.virtdata.core.templates; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class CapturePointParserTest { @@ -31,9 +33,50 @@ public class CapturePointParserTest { assertThat(result).isEqualTo( new CapturePointParser.Result( "string with capture1", - List.of(CapturePoint.of("capture1")) + new CapturePoints(List.of(CapturePoint.of("capture1"))) ) ); } + @Test + public void testCapturePointWithCast() { + CapturePointParser cpp = new CapturePointParser(); + CapturePointParser.Result result = cpp.apply("string with [(Number)capture1]"); + assertThat(result).isEqualTo( + new CapturePointParser.Result( + "string with capture1", + new CapturePoints(List.of(CapturePoint.of(Number.class,"capture1", "capture1"))) + ) + ); + CapturePoint cp1 = result.getCaptures().get(0); + assertThat(cp1.toString()).isEqualTo("[(Number)capture1]"); + + assertThat(cp1.valueOf(Long.valueOf(42L))).isEqualTo(42L); + assertThatThrownBy(() -> cp1.valueOf("42L")).hasMessageContaining("Cannot cast java.lang.String to java.lang.Number"); + } + + @Test + public void testCaptureWildcard() { + CapturePointParser cpp = new CapturePointParser(); + CapturePointParser.Result result = cpp.apply("select [*] from foobarbaz"); + assertThat(result).isEqualTo( + new CapturePointParser.Result( + "select * from foobarbaz", + new CapturePoints(List.of(CapturePoint.of(Object.class, "*", "*"))) + ) + ); + } + + @Test + public void testMapEquivalence() { + CapturePointParser cpp = new CapturePointParser(); + CapturePointParser.Result result = cpp.parse(Map.of("asname","(Integer) varname")); + assertThat(result).isEqualTo( + new CapturePointParser.Result( + "varname", + new CapturePoints(List.of(CapturePoint.of(Integer.class, "varname", "asname"))) + ) + ); + + } } diff --git a/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointTest.java b/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointTest.java index ba6498d4a..59cab7ba5 100644 --- a/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointTest.java +++ b/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointTest.java @@ -29,10 +29,10 @@ public class CapturePointTest { CapturePointParser cpp = new CapturePointParser(); assertThat(cpp.apply("test [point1] [point2 as alias3]")).isEqualTo( new CapturePointParser.Result("test point1 point2", - List.of( + new CapturePoints(List.of( CapturePoint.of("point1"), CapturePoint.of("point2","alias3") - )) + ))) ); } @@ -40,7 +40,7 @@ public class CapturePointTest { public void testBypass() { CapturePointParser cpp = new CapturePointParser(); assertThat(cpp.apply("")).isEqualTo( - new CapturePointParser.Result("", List.of()) + new CapturePointParser.Result("", new CapturePoints()) ); } diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_activity_error.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_activity_error.java index 0868a8d5f..6de247aa6 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_activity_error.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_activity_error.java @@ -20,6 +20,7 @@ import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContaine import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand; import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController; +import org.jetbrains.annotations.NotNull; import java.io.PrintWriter; @@ -53,12 +54,9 @@ public class NB_activity_error extends NBBaseCommand { */ @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var activitydef1 = Map.of("alias", "activity_error", - "driver", "diag", - "cycles", "0..1500000", - "threads", "1", - "targetrate", "10", - "op", "log: modulo=1" + Map activitydef1 = Map.of( + "alias", "activity_error", "driver", "diag", "cycles", "0..1500000", "threads", "1", + "targetrate", "10", "op", "log: modulo=1" ); stdout.write("starting activity activity_error"); controller.start(activitydef1); diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_await_finished.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_await_finished.java index d81feb2ab..9d555b9b0 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_await_finished.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_await_finished.java @@ -20,6 +20,7 @@ import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitie import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer; import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand; +import org.jetbrains.annotations.NotNull; import java.io.PrintWriter; @@ -50,13 +51,9 @@ public class NB_await_finished extends NBBaseCommand { */ @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var activitydef1 = Map.of( - "alias", "activity_to_await", - "driver", "diag", - "cycles", "0..1500", - "threads", "1", - "targetrate", "500", - "op", "noop" + Map activitydef1 = Map.of( + "alias", "activity_to_await", "driver", "diag", "cycles", "0..1500", "threads", "1", + "targetrate", "500", "op", "noop" ); stdout.println("starting activity activity_to_await"); controller.start(activitydef1); diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_basicdiag.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_basicdiag.java index fd7ea98fd..05c27264c 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_basicdiag.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_basicdiag.java @@ -45,9 +45,8 @@ public class NB_basicdiag extends NBBaseCommand { */ @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var basic_diag = params.withOverrides( - Map.of("alias","basic_diag","driver","diag") - ); + NBCommandParams basic_diag = params.withOverrides( + Map.of("alias", "basic_diag", "driver", "diag")); stdout.println("starting activity basic_diag"); controller.start(basic_diag); stdout.println("stopping activity basic_diag"); diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_cocycledelay_bursty_backup.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_cocycledelay_bursty_backup.java index 88c902eeb..5a563b157 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_cocycledelay_bursty_backup.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_cocycledelay_bursty_backup.java @@ -22,6 +22,7 @@ import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter; import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge; import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController; +import org.jetbrains.annotations.NotNull; import java.io.PrintWriter; @@ -86,13 +87,9 @@ public class NB_cocycledelay_bursty_backup extends NBBaseCommand { */ @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var co_cycle_delay_bursty = Map.of( - "alias", "co_cycle_delay_bursty", - "driver", "diag", - "cycles", "0..1000000", - "threads", "1", - "cyclerate", "1000,1.5", - "op", "diagrate: diagrate=500" + Map co_cycle_delay_bursty = Map.of( + "alias", "co_cycle_delay_bursty", "driver", "diag", "cycles", "0..1000000", "threads", + "1", "cyclerate", "1000,1.5", "op", "diagrate: diagrate=500" ); diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_extension_csvoutput.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_extension_csvoutput.java index 107cd5dde..c3cbef0b2 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_extension_csvoutput.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_extension_csvoutput.java @@ -30,12 +30,6 @@ public class NB_extension_csvoutput extends NBBaseCommand { super(parentComponent, scenarioName); } - /** 
      {@code
-     * var csvlogger = csvoutput.open("logs/csvoutputtestfile.csv","header1","header2");
-     *
-     * csvlogger.write({"header1": "value1","header2":"value2"});
-     * }
      - */ @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { return null; diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_linkedinput.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_linkedinput.java index 7c6aba008..72be1d8af 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_linkedinput.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_linkedinput.java @@ -20,6 +20,7 @@ import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContaine import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController; import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand; +import org.jetbrains.annotations.NotNull; import java.io.PrintWriter; import java.io.Reader; @@ -30,48 +31,13 @@ public class NB_linkedinput extends NBBaseCommand { super(parentComponent, scenarioName); } - /** - * 
      {@code
-     * var leader = {
-     *     driver: 'diag',
-     *     alias: 'leader',
-     *     targetrate: '10000',
-     *     op: 'log:level=info'
-     * };
-     *
-     * var follower = {
-     *     driver: 'diag',
-     *     alias: 'follower',
-     *     // linkinput: 'leader',
-     *     op: 'log:level=INFO'
-     * };
-     *
-     * scenario.start(leader);
-     * print("started leader");
-     * scenario.start(follower);
-     * print("started follower");
-     *
-     * scenario.waitMillis(500);
-     *
-     * scenario.stop(leader);
-     * print("stopped leader");
-     * scenario.stop(follower);
-     * print("stopped follower");
-     * }
      - */ @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var leader = Map.of( - "driver", "diag", - "alias", "leader", - "targetrate", "10000", - "op", "log:level=info" - ); + Map leader = Map.of( + "driver", "diag", "alias", "leader", "targetrate", "10000", "op", "log:level=info"); - var follower = Map.of( - "driver", "diag", - "alias", "follower", - "op", "log:level=INFO" + Map follower = Map.of( + "driver", "diag", "alias", "follower", "op", "log:level=INFO" ); diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_start_stop_diag.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_start_stop_diag.java index 4a4a4f6a8..f78ace801 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_start_stop_diag.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_start_stop_diag.java @@ -20,6 +20,7 @@ import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitie import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer; import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand; +import org.jetbrains.annotations.NotNull; import java.io.PrintWriter; @@ -59,14 +60,10 @@ public class NB_start_stop_diag extends NBBaseCommand { @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var activitydef = Map.of( - "alias" , "teststartstopdiag", - "driver" , "diag", - "cycles" , "0..1000000000", - "threads" , "5", - "interval" , "2000", - "op" , "noop", - "rate" , "5"); + Map activitydef = Map.of( + "alias", "teststartstopdiag", "driver", "diag", "cycles", "0..1000000000", "threads", + "5", "interval", "2000", "op", "noop", "rate", "5" + ); stdout.println("starting activity teststartstopdiag"); controller.start(activitydef); diff --git a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_threadchange.java b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_threadchange.java index 521c6c06d..ae149ec04 100644 --- a/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_threadchange.java +++ b/nbr-demos/src/test/java/io/nosqlbench/nbr/examples/injava/NB_threadchange.java @@ -16,6 +16,7 @@ package io.nosqlbench.nbr.examples.injava; +import io.nosqlbench.engine.api.activityapi.core.Activity; import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer; import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand; import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController; @@ -46,7 +47,8 @@ public class NB_threadchange extends NBBaseCommand { @Override public Object invoke(NBCommandParams params, PrintWriter stdout, PrintWriter stderr, Reader stdin, ContainerActivitiesController controller) { - var activity = controller.start("driver=diag;alias=threadchange;cycles=0..60000;threads=1;interval=2000;op='noop';rate=1000"); + Activity activity = controller.start( + "driver=diag;alias=threadchange;cycles=0..60000;threads=1;interval=2000;op='noop';rate=1000"); activity.getActivityDef().setThreads(1); stdout.println("threads now " + activity.getActivityDef().getThreads()); stdout.println("waiting 500 ms"); diff --git a/nbr/src/main/resources/verify.md b/nbr/src/main/resources/verify.md new file mode 100644 index 000000000..71ec869b7 --- /dev/null +++ b/nbr/src/main/resources/verify.md @@ -0,0 +1,164 @@ +# Data Verification + +NoSQLBench has the capability to verify data that is the result of an operation by comparing +it with reference data. The reference data is simply the same data which you might use to generate +data for upsert, modeled with standard bindings. Thus, you can assert the correctness of any (whole +or part of) data within a database for which you have data bindings. For the most common case -- +writing and then verifying data, the same bindings are used for both phases. + +The verifier is a generalized capability that can be used with any operation which produces +a value. At it's core, the verifier is a scriptable block can receive the result value in the +context parameter `result`, with the type of value determined by the op implementation and settings. + + +If you specify `verify` as the op type with the cqld4 adapter, then it will do the following: + +1. Presume that the op structure is effectively a read operation, meaning it will produce a result + that can be used for comparison. The operation must have a single row in the result set. If there + is no row, then the row fails validation. The same happens if there is more than one row. +2. Using the provided bindings, re-generate the data separately which was expected to be in the + database. This is called _reference data_ herein. +3. Verify that the values returned from the database are the same as the reference data, and throw a + ResultVerificationError for each mis-matching row. + +Alternately, you can add the verify and compare options to any regular CQL statement (Simple, Raw, +Prepared) in order to enable this verification logic. + +## Verification Options + +These options may be attached to an op template directly as op fields, or they may be passed as +activity parameters. NOTE: passing them as activity parameters will only work if all of the active +operations are compatible with the verify options. + +- **verify** - an optional modifier of fields to verify for a statement. If this parameter is not + provided, then it is presumed to be `*` by default. This is a string which consists of + comma-separate values. This parameter is useful if you have a set of default bindings and want to + specify which subset of them will be used just for this statement. Each form modifies the list of + fields to verify incrementally, and multiple options are allowed: +- `*` : If the value is `*`, then all the bindings that are visible for the statement will be used + as expected values. +- `-`, `-field2` : If it is a word that starts with `-`, like `-field2`, then the name after the + dash is removed from the list of fields to verify. +- `+`, `+field3` : If it is a word that starts with a `+`, like `+field3`, or a simple word, then + the field is added to the list of fields to verify. +- `f->b` : If any of the added fields is in the form `f->b`, then it is taken as a mapping from the + field name `f` in the schema to a binding `b`. + +For example, + +```yaml +# example op template +ops: + op1: + readit: "select ....TBD" + bindings: + a: .. + b: .. + c: .. + verify: "*,-b" +``` + +means _verify all fields from the bindings except `b`_, using the default validation method. + +- **compare** - what to verify, naming structure, values, etc. each of these is additive, and + multiple can be specified. + - all (the default) - A synonym for fields AND values + - fields - A synonym for rowfields AND reffields (full set equivalence) + - rowfields - Verify that the result field names include every reference field name. + - reffields - Verify that the reference field names include every result field name. + - values - Verify that all the pair-wise fields have equal values, according to the + type-specific `.equals(...)` method for the data type identified in the row metadata by field + name. + +For example, + +```yaml +# example op template +ops: + op1: + readit: "select ....TBD" + bindings: + a: .. + b: .. + c: .. + verify: "*,-b" + compare: reffields +``` + +means _ensure reference fields are present in result fields by name but do not throw an error if +there are more result fields present, and do not compare values of same-named fields_. + +## Verification Results + +### Errors + +The data bindings are used to generate the expected values that would be used for an upsert. Each +row is verified according to these values, and any discrepancy is treated as an error that can be +counted, logged, etc. If you want to simply count the occurences instead of fast-failing an activity +when a row is unverified, then you simply need to modify the error handler for that activity: ' +errors=unverified=count'. + +The default error handling behavior will cause an exception to be thrown and NoSQLBench +will exit by default. If you wish for something less dramatic, then +wire the unverified category to something else: + + errors=...,unverified->count + +or + + errors=...,unverified->warn + +### Metrics + +The cqlverify activity type adds some verification-specific metrics: + +- alias.verified_results - A counter for how many results were verified +- alias.unverified_results - A counter for how many results were not verified +- alias.verified_values - A counter for how many field values were verified +- alias.unverified_values - A counter for how many field values were unverified + +## Advanced Usage + +The verify capability does not retain logged data for verification. Still, it is able to compare +data as if it had a separate physical data set to compare to. This is possible only because +virtdata (the data generation layer of NoSQLBench) can provide realistic views of virtual datasets +on the fly. + +### Avoid Random data + +That means, however, that you must avoid using the non-stable data mapping functions when writing +data. The rule of thumb is to avoid using any data mapping functions containing the word "Random". +Binding functions with `random` in their name behave differently from others in that they will not +produce stable results. Their initialization vector is external to the function definition, such as +when using the system _random_ functions. + +> Some will bristle at this misuse of the terms, but connotatively they work well for most +> other users. Actually, all of the algorithms used by +> virtdata are __NOT__ truly random, and are deterministic in some way. However, some rely on +> an initialization vector which is not self-contained within the function definition. As such, +> these functions are not pure functions in practice and thus may not be relied upon to return the +> same result from session to session. The word _random_ in virtdata binding functions indicates +> that a function is non-determinstic in some cases. As long as you avoid these functions, +> you can rely on stable generated data from session to session. + +### Single vs Multiple Results + +TBD + +### Paging vs Non-Paging verification + +TBD + +### Example activity definitions + +Write 100K cycles of telemetry data + +> TBD ... run driver=cql alias=writesome workload=cql-iot tags=group:write cycles=100000 host=... + +Verify the the same 100K cycles of telemetry data + +> TBD ... run driver=cqlverify alias=verify workload=cql-iot tags=group:verify cycles=100000 + +To see how these examples work, TBD + +