diff --git a/nb-adapters/adapter-dataapi/pom.xml b/nb-adapters/adapter-dataapi/pom.xml index 2630f0d61..cbf0be2c0 100644 --- a/nb-adapters/adapter-dataapi/pom.xml +++ b/nb-adapters/adapter-dataapi/pom.xml @@ -49,7 +49,7 @@ com.datastax.astra astra-db-java - 2.1.4 + 2.3.2 diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 42efd195d..306e76d05 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -26,10 +26,46 @@ import io.nosqlbench.engine.api.templating.TypeAndTarget; import io.nosqlbench.nb.api.components.core.NBComponent; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; +import java.util.Map; import java.util.function.LongFunction; public class DataApiOpMapper implements OpMapper { private static final Logger logger = LogManager.getLogger(DataApiOpMapper.class); + + // Maps a deprecated op type to its replacement, or null if there is no replacement. + private static final Map DEPRECATED_OP_TYPES; + static { + DEPRECATED_OP_TYPES = new HashMap<>(); + // admin ops: + DEPRECATED_OP_TYPES.put(DataApiOpType.create_collection_with_class, DataApiOpType.db_create_collection); + DEPRECATED_OP_TYPES.put(DataApiOpType.create_collection, DataApiOpType.db_create_collection); + DEPRECATED_OP_TYPES.put(DataApiOpType.delete_collection, DataApiOpType.db_delete_collection); + // db-admin ops: + // in-database ops: + // in-collection ops: + DEPRECATED_OP_TYPES.put(DataApiOpType.insert_one, DataApiOpType.collection_insert_one); + DEPRECATED_OP_TYPES.put(DataApiOpType.insert_one_vector, DataApiOpType.collection_insert_one); + DEPRECATED_OP_TYPES.put(DataApiOpType.insert_many, DataApiOpType.collection_insert_many); + DEPRECATED_OP_TYPES.put(DataApiOpType.delete_all, DataApiOpType.collection_delete_many); + DEPRECATED_OP_TYPES.put(DataApiOpType.delete_many, DataApiOpType.collection_delete_many); + DEPRECATED_OP_TYPES.put(DataApiOpType.delete_one, DataApiOpType.collection_delete_one); + DEPRECATED_OP_TYPES.put(DataApiOpType.find_one_and_delete, DataApiOpType.collection_find_one_and_delete); + DEPRECATED_OP_TYPES.put(DataApiOpType.find, DataApiOpType.collection_find); + DEPRECATED_OP_TYPES.put(DataApiOpType.find_vector, DataApiOpType.collection_find); + DEPRECATED_OP_TYPES.put(DataApiOpType.find_vector_filter, DataApiOpType.collection_find); + DEPRECATED_OP_TYPES.put(DataApiOpType.find_by_id, DataApiOpType.collection_find_one); + DEPRECATED_OP_TYPES.put(DataApiOpType.find_distinct, null); // not a Data API primitive, it's client-emulated + DEPRECATED_OP_TYPES.put(DataApiOpType.find_one, DataApiOpType.collection_find_one); + DEPRECATED_OP_TYPES.put(DataApiOpType.update_one, DataApiOpType.collection_update_one); + DEPRECATED_OP_TYPES.put(DataApiOpType.find_one_and_update, DataApiOpType.collection_find_one_and_update); + DEPRECATED_OP_TYPES.put(DataApiOpType.replace_one, null); // not a Data API primitive, it's client-emulated + DEPRECATED_OP_TYPES.put(DataApiOpType.find_one_and_replace, DataApiOpType.collection_find_one_and_replace); + DEPRECATED_OP_TYPES.put(DataApiOpType.update_many, DataApiOpType.collection_update_many); + DEPRECATED_OP_TYPES.put(DataApiOpType.count_documents, DataApiOpType.collection_count_documents); + DEPRECATED_OP_TYPES.put(DataApiOpType.estimated_document_count, DataApiOpType.collection_estimated_document_count); + } + private final DataApiDriverAdapter adapter; public DataApiOpMapper(DataApiDriverAdapter dataApiDriverAdapter) { @@ -47,7 +83,18 @@ public class DataApiOpMapper implements OpMapper { "collection" ); logger.debug(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); + if (DEPRECATED_OP_TYPES.containsKey(typeAndTarget.enumId)) { + DataApiOpType replacement = DEPRECATED_OP_TYPES.get(typeAndTarget.enumId); + if (replacement != null) { + logger.warn(() -> "Op type '" + typeAndTarget.enumId + "' is deprecated; please switch to '" + + replacement + "'. Op template: '" + op.getName() + "'"); + } else { + logger.warn(() -> "Op type '" + typeAndTarget.enumId + "' is discontinued and will be removed" + + " in a future release. Op template: '" + op.getName() + "'"); + } + } return switch (typeAndTarget.enumId) { + // TO-DO OPS // admin ops: case create_database -> new DataApiCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); case list_databases -> new DataApiListDatabasesOpDispenser(adapter, op, typeAndTarget.targetFunction); @@ -58,34 +105,60 @@ public class DataApiOpMapper implements OpMapper { case list_namespaces -> new DataApiListNamespacesOpDispenser(adapter, op, typeAndTarget.targetFunction); case drop_namespace -> new DataApiDropNamespaceOpDispenser(adapter, op, typeAndTarget.targetFunction); // in-database ops: - case create_collection -> new DataApiCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete_collection -> new DataApiDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); case list_collections -> new DataApiListCollectionsOpDispenser(adapter, op, typeAndTarget.targetFunction); case list_collection_names -> new DataApiListCollectionNamesOpDispenser(adapter, op, typeAndTarget.targetFunction); - case create_collection_with_class -> new DataApiCreateCollectionWithClassOpDispenser(adapter, op, typeAndTarget.targetFunction); + + // LEGACY OPS + // admin ops: + // db-admin ops: + // in-database ops: + case create_collection -> new DataApiLegacyCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + case create_collection_with_class -> new DataApiLegacyCreateCollectionWithClassOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_collection -> new DataApiLegacyDeleteCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); // in-collection ops: - case insert_many -> new DataApiCollectionInsertManyOpDispenser(adapter, op, typeAndTarget.targetFunction); case insert_one -> new DataApiCollectionInsertOneOpDispenser(adapter, op, typeAndTarget.targetFunction); - case insert_one_vector -> new DataApiCollectionInsertOneVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find -> new DataApiCollectionFindOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_one -> new DataApiCollectionFindOneOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_one_and_delete -> new DataApiCollectionFindOneAndDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_one_and_update -> new DataApiCollectionFindOneAndUpdateOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_vector -> new DataApiCollectionFindVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_vector_filter -> new DataApiCollectionFindVectorFilterOpDispenser(adapter, op, typeAndTarget.targetFunction); - case update_one -> new DataApiCollectionUpdateOneOpDispenser(adapter, op, typeAndTarget.targetFunction); - case update_many -> new DataApiCollectionUpdateManyOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete_one -> new DataApiCollectionDeleteOneOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete_many -> new DataApiCollectionDeleteManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert_one_vector -> new DataApiCollectionLegacyInsertOneVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert_many -> new DataApiCollectionLegacyInsertManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_one -> new DataApiCollectionLegacyDeleteOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_many -> new DataApiCollectionLegacyDeleteManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_all -> new DataApiCollectionLegacyDeleteAllOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one_and_delete -> new DataApiCollectionLegacyFindOneAndDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find -> new DataApiCollectionLegacyFindOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_vector -> new DataApiCollectionLegacyFindVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_vector_filter -> new DataApiCollectionLegacyFindVectorFilterOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_by_id -> new DataApiCollectionLegacyFindByIdOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_distinct -> new DataApiCollectionLegacyFindDistinctOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one -> new DataApiCollectionLegacyFindOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case update_one -> new DataApiCollectionLegacyUpdateOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one_and_update -> new DataApiCollectionLegacyFindOneAndUpdateOpDispenser(adapter, op, typeAndTarget.targetFunction); + case replace_one -> new DataApiCollectionLegacyReplaceOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one_and_replace -> new DataApiCollectionLegacyFindOneAndReplaceOpDispenser(adapter, op, typeAndTarget.targetFunction); + case update_many -> new DataApiCollectionLegacyUpdateManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case count_documents -> new DataApiCollectionLegacyCountDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); case estimated_document_count -> new DataApiCollectionEstimatedDocumentCountOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_by_id -> new DataApiCollectionFindByIdOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_distinct -> new DataApiCollectionFindDistinctOpDispenser(adapter, op, typeAndTarget.targetFunction); - case count_documents -> new DataApiCollectionCountDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); - case replace_one -> new DataApiCollectionReplaceOneOpDispenser(adapter, op, typeAndTarget.targetFunction); - case find_one_and_replace -> new DataApiCollectionFindOneAndReplaceOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete_all -> new DataApiCollectionDeleteAllOpDispenser(adapter, op, typeAndTarget.targetFunction); + + // NEW-STYLE OPS + // in-database ops: + case db_create_collection -> new DataApiDbCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + case db_delete_collection -> new DataApiDbDeleteCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + // in-collection ops: + case collection_insert_one -> new DataApiCollectionInsertOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_insert_many -> new DataApiCollectionInsertManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_find -> new DataApiCollectionFindOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_find_one -> new DataApiCollectionFindOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_find_and_rerank -> new DataApiCollectionFindAndRerankOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_update_one -> new DataApiCollectionUpdateOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_find_one_and_update -> new DataApiCollectionFindOneAndUpdateOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_delete_many -> new DataApiCollectionDeleteManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_delete_one -> new DataApiCollectionDeleteOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_find_one_and_delete -> new DataApiCollectionFindOneAndDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_find_one_and_replace -> new DataApiCollectionFindOneAndReplaceOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_update_many -> new DataApiCollectionUpdateManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_estimated_document_count -> + new DataApiCollectionEstimatedDocumentCountOpDispenser(adapter, op, typeAndTarget.targetFunction); + case collection_count_documents -> new DataApiCollectionCountDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index 5f579c203..657385f92 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -20,6 +20,9 @@ import com.datastax.astra.client.DataAPIClient; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.admin.AstraDBAdmin; import com.datastax.astra.client.admin.DatabaseAdmin; +// TODO: remove this - it's to log Data API requests and responses +import com.datastax.astra.client.core.options.DataAPIClientOptions; + import io.nosqlbench.adapters.api.activityimpl.uniform.BaseSpace; import io.nosqlbench.nb.api.config.standard.ConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfigModel; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionCountDocumentsOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionCountDocumentsOpDispenser.java index 63cf997e6..2398817d7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionCountDocumentsOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionCountDocumentsOpDispenser.java @@ -26,6 +26,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; +import java.util.Optional; public class DataApiCollectionCountDocumentsOpDispenser extends DataApiOpDispenser { private static final Logger logger = LogManager.getLogger(DataApiCollectionCountDocumentsOpDispenser.class); @@ -40,7 +41,13 @@ public class DataApiCollectionCountDocumentsOpDispenser extends DataApiOpDispens return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); - int upperBound = op.getAsRequiredFunction("upperbound", Integer.class).apply(l); + Optional> ubf = op.getAsOptionalFunction("upper_bound", Integer.class); + Optional upperBound; + if (ubf.isPresent()) { + upperBound = Optional.of(ubf.get().apply(l)); + } else { + upperBound = Optional.empty(); + } return new DataApiCollectionCountDocumentsOp( db, diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteOneOpDispenser.java index eba8b453a..98db0b751 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteOneOpDispenser.java @@ -55,23 +55,13 @@ public class DataApiCollectionDeleteOneOpDispenser extends DataApiOpDispenser { private CollectionDeleteOneOptions getCollectionDeleteOneOptions(ParsedOp op, long l) { CollectionDeleteOneOptions options = new CollectionDeleteOneOptions(); - Sort sort = getSortFromOp(op, l); - float[] vector = getVectorFromOp(op, l); - if (sort != null) { - options = (vector != null) ? options.sort(Sort.vector(vector), sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(Sort.vector(vector)); + Sort[] sorts = getSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); } return options; } - private float[] getVectorFromOp(ParsedOp op, long l) { - if (op.isDefined("vector")) { - return getVectorValues(op.get("vector", l)); - } - return null; - } - @Override public DataApiBaseOp getOp(long cycle) { return opFunction.apply(cycle); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindAndRerankOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindAndRerankOpDispenser.java new file mode 100644 index 000000000..f4761c7c1 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindAndRerankOpDispenser.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.collections.commands.options.CollectionFindAndRerankOptions; +import com.datastax.astra.client.core.query.Sort; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Projection; +import com.datastax.astra.client.core.rerank.RerankServiceOptions; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindAndRerankOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; +import java.util.Map; +import java.util.Optional; + +public class DataApiCollectionFindAndRerankOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionFindAndRerankOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionFindAndRerankOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + CollectionFindAndRerankOptions options = getCollectionFindAndRerankOptions(op, l); + + return new DataApiCollectionFindAndRerankOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; + } + + private CollectionFindAndRerankOptions getCollectionFindAndRerankOptions(ParsedOp op, long l) { + CollectionFindAndRerankOptions options = new CollectionFindAndRerankOptions(); + Sort sort = getFindAndRerankSortFromOp(op, l); + options = options.sort(sort); + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + Optional limit = getLimitFromOp(op, l); + if (limit.isPresent()) { + options = options.limit(limit.get()); + } + Optional includeSortVector = getIncludeSortVectorFromOp(op, l); + if (includeSortVector.isPresent()) { + options = options.includeSortVector(includeSortVector.get()); + } + Optional includeScores = getIncludeScoresFromOp(op, l); + if (includeScores.isPresent()) { + options = options.includeScores(includeScores.get()); + } + Optional rerankQuery = getRerankQueryFromOp(op, l); + if (rerankQuery.isPresent()) { + options = options.rerankQuery(rerankQuery.get()); + } + Optional rerankOn = getRerankOnFromOp(op, l); + if (rerankOn.isPresent()) { + options = options.rerankOn(rerankOn.get()); + } + Optional hybridLimits = getHybridLimitsFromOp(op, l); + if (hybridLimits.isPresent()) { + Object hl = hybridLimits.get(); + if (hl instanceof Integer i) { + options = options.hybridLimits(i); + } else { + @SuppressWarnings("unchecked") + Map hlMap = (Map) hl; + options = options.hybridLimits(hlMap); + } + } + Optional rerankService = getRerankServiceFromOp(op, l); + if (rerankService.isPresent()) { + options = options.rerankService(rerankService.get()); + } + // TEMP + RerankServiceOptions rop = new RerankServiceOptions(); + rop.provider("providerZ"); + rop.modelName("modelNameZ"); + options = options.rerankService(rop); + // END TEMP + + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndDeleteOpDispenser.java index fd8fc11a5..17aba0f00 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndDeleteOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndDeleteOpDispenser.java @@ -56,9 +56,9 @@ public class DataApiCollectionFindOneAndDeleteOpDispenser extends DataApiOpDispe private CollectionFindOneAndDeleteOptions getCollectionFindOneAndDeleteOptions(ParsedOp op, long l) { CollectionFindOneAndDeleteOptions options = new CollectionFindOneAndDeleteOptions(); - Sort sort = getSortFromOp(op, l); - if (sort != null) { - options = options.sort(sort); + Sort[] sorts = getSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndReplaceOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndReplaceOpDispenser.java index d8c8937c9..a959baeee 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndReplaceOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndReplaceOpDispenser.java @@ -30,8 +30,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class DataApiCollectionFindOneAndReplaceOpDispenser extends DataApiOpDispenser { @@ -48,14 +46,13 @@ public class DataApiCollectionFindOneAndReplaceOpDispenser extends DataApiOpDisp Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); CollectionFindOneAndReplaceOptions options = getCollectionFindOneAndReplaceOptions(op, l); - LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); - LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + Document replacement = getReplacementFromOp(op, l); return new DataApiCollectionFindOneAndReplaceOp( db, db.getCollection(targetFunction.apply(l)), filter, - docFunc.apply(l), + replacement, options ); }; @@ -63,29 +60,21 @@ public class DataApiCollectionFindOneAndReplaceOpDispenser extends DataApiOpDisp private CollectionFindOneAndReplaceOptions getCollectionFindOneAndReplaceOptions(ParsedOp op, long l) { CollectionFindOneAndReplaceOptions options = new CollectionFindOneAndReplaceOptions(); - Sort sort = getSortFromOp(op, l); - if (op.isDefined("vector")) { - float[] vector = getVectorValues(op, l); - if (sort != null) { - options = vector != null ? options.sort(Sort.vector(vector), sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(Sort.vector(vector)); - } + Sort[] sorts = getSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { options = options.projection(projection); } - Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); - if (upsertFunction.isPresent()) { - options = options.upsert(upsertFunction.get().apply(l)); + Boolean upsert = getUpsertFromOp(op, l); + if (upsert != null) { + options = options.upsert(upsert); } - if (op.isDefined("returnDocument")) { - options = switch ((String) op.get("returnDocument", l)) { - case "after" -> options.returnDocument(ReturnDocument.AFTER); - case "before" -> options.returnDocument(ReturnDocument.BEFORE); - default -> throw new RuntimeException("Invalid returnDocument value: " + op.get("returnDocument", l)); - }; + ReturnDocument returnDocument = getReturnDocumentFromOp(op, l); + if (returnDocument != null){ + options = options.returnDocument(returnDocument); } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndUpdateOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndUpdateOpDispenser.java index 2bbd2df21..755b2f5a0 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndUpdateOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneAndUpdateOpDispenser.java @@ -18,7 +18,11 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Sort; import com.datastax.astra.client.collections.commands.Update; +import com.datastax.astra.client.collections.commands.options.CollectionDeleteOneOptions; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneAndUpdateOptions; + import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindOneAndUpdateOp; @@ -40,17 +44,32 @@ public class DataApiCollectionFindOneAndUpdateOpDispenser extends DataApiOpDispe return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); - Update update = getUpdates(op, l); + Update update = getUpdateFromOp(op, l); + CollectionFindOneAndUpdateOptions options = getCollectionFindOneAndUpdateOptions(op, l); return new DataApiCollectionFindOneAndUpdateOp( db, db.getCollection(targetFunction.apply(l)), filter, - update + update, + options ); }; } + private CollectionFindOneAndUpdateOptions getCollectionFindOneAndUpdateOptions(ParsedOp op, long l) { + CollectionFindOneAndUpdateOptions options = new CollectionFindOneAndUpdateOptions(); + Sort[] sorts = getSortFromOp(op, l); + Boolean upsert = getUpsertFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + if ( upsert != null ){ + options = options.upsert(upsert); + } + return options; + } + @Override public DataApiBaseOp getOp(long cycle) { return opFunction.apply(cycle); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneOpDispenser.java index 23379ff36..75553269c 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOneOpDispenser.java @@ -28,6 +28,7 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiCollectionFindOneOpDispenser extends DataApiOpDispenser { @@ -54,18 +55,18 @@ public class DataApiCollectionFindOneOpDispenser extends DataApiOpDispenser { private CollectionFindOneOptions getCollectionFindOneOptions(ParsedOp op, long l) { CollectionFindOneOptions options = new CollectionFindOneOptions(); - Sort sort = getSortFromOp(op, l); - float[] vector = getVectorValues(op, l); - if (sort != null) { - options = vector != null ? options.sort(Sort.vector(vector), sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(Sort.vector(vector)); + Sort[] sorts = getSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { options = options.projection(projection); } - options.includeSimilarity(true); + Optional includeSimilarity = getIncludeSimilarityFromOp(op, l); + if (includeSimilarity.isPresent()) { + options.includeSimilarity(includeSimilarity.get()); + } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOpDispenser.java index 561ee4b61..602ff3de9 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindOpDispenser.java @@ -55,34 +55,29 @@ public class DataApiCollectionFindOpDispenser extends DataApiOpDispenser { private CollectionFindOptions getCollectionFindOptions(ParsedOp op, long l) { CollectionFindOptions options = new CollectionFindOptions(); - Sort sort = getSortFromOp(op, l); - if (op.isDefined("vector")) { - float[] vector = getVectorValues(op, l); - if (sort != null) { - options = vector != null ? options.sort(Sort.vector(vector), sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(Sort.vector(vector)); - } + Sort[] sorts = getSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { options = options.projection(projection); } - Optional> limitFunction = op.getAsOptionalFunction("limit", Integer.class); - if (limitFunction.isPresent()) { - options = options.limit(limitFunction.get().apply(l)); + Optional limit = getLimitFromOp(op, l); + if (limit.isPresent()) { + options = options.limit(limit.get()); } - Optional> skipFunction = op.getAsOptionalFunction("skip", Integer.class); - if (skipFunction.isPresent()) { - options = options.skip(skipFunction.get().apply(l)); + Optional skip = getSkipFromOp(op, l); + if (skip.isPresent()) { + options = options.skip(skip.get()); } - Optional> includeSimilarityFunction = op.getAsOptionalFunction("includeSimilarity", Boolean.class); - if (includeSimilarityFunction.isPresent()) { - options.includeSimilarity(includeSimilarityFunction.get().apply(l)); + Optional includeSortVector = getIncludeSortVectorFromOp(op, l); + if (includeSortVector.isPresent()) { + options = options.includeSortVector(includeSortVector.get()); } - Optional> pageStateFunction = op.getAsOptionalFunction("pageState", String.class); - if (pageStateFunction.isPresent()) { - options.pageState(pageStateFunction.get().apply(l)); + Optional includeSimilarity = getIncludeSimilarityFromOp(op, l); + if (includeSimilarity.isPresent()) { + options.includeSimilarity(includeSimilarity.get()); } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertManyOpDispenser.java index 659aa90aa..040b4c677 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertManyOpDispenser.java @@ -42,8 +42,7 @@ public class DataApiCollectionInsertManyOpDispenser extends DataApiOpDispenser { private LongFunction createOpFunction(ParsedOp op) { return (l) -> { - List documents = new ArrayList<>(); - op.getAsRequiredFunction("documents", List.class).apply(l).forEach(o -> documents.add(Document.parse(o.toString()))); + List documents = getDocumentsFromOp(op, l); return new DataApiCollectionInsertManyOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), @@ -55,21 +54,15 @@ public class DataApiCollectionInsertManyOpDispenser extends DataApiOpDispenser { private CollectionInsertManyOptions getCollectionInsertManyOptions(ParsedOp op, long l) { CollectionInsertManyOptions options = new CollectionInsertManyOptions(); - Optional> optionsFunction = op.getAsOptionalFunction("options", Map.class); - if (optionsFunction.isPresent()) { - Map optionFields = optionsFunction.get().apply(l); - for(Map.Entry entry: optionFields.entrySet()) { - switch(entry.getKey()) { - case "chunkSize"-> - options = options.chunkSize(Integer.parseInt(entry.getValue())); - case "concurrency" -> - options = options.concurrency(Integer.parseInt(entry.getValue())); - case "ordered" -> - options = options.ordered(Boolean.parseBoolean(entry.getValue())); - case "timeout" -> - options = options.timeout(Integer.parseInt(entry.getValue())); - } - } + options = options.concurrency(1); // hardcoded: avoid client concurrency, let NB use its own. + Integer chunkSize = getChunkSizeFromOp(op, l); + if (chunkSize != null) { + logger.warn(() -> "Setting insertion 'chunk_size'. This is discouraged, as it usually signals an intention to not have 1:1 between ops and HTTP requests."); + options = options.chunkSize(chunkSize); + } + Boolean ordered = getOrderedFromOp(op, l); + if (ordered != null) { + options = options.ordered(ordered); } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneOpDispenser.java index 072bcb1a4..4ea489e34 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneOpDispenser.java @@ -37,13 +37,12 @@ public class DataApiCollectionInsertOneOpDispenser extends DataApiOpDispenser { } private LongFunction createOpFunction(ParsedOp op) { - LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); - LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); return (l) -> { + Document document = getDocumentFromOp(op, l); return new DataApiCollectionInsertOneOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), - docFunc.apply(l) + document ); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyCountDocumentsOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyCountDocumentsOpDispenser.java new file mode 100644 index 000000000..667b407ac --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyCountDocumentsOpDispenser.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyCountDocumentsOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyCountDocumentsOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyCountDocumentsOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyCountDocumentsOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + int upperBound = op.getAsRequiredFunction("upperbound", Integer.class).apply(l); + + return new DataApiCollectionLegacyCountDocumentsOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + upperBound + ); + }; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteAllOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteAllOpDispenser.java similarity index 71% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteAllOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteAllOpDispenser.java index 569a0685e..721ef5281 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionDeleteAllOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteAllOpDispenser.java @@ -19,27 +19,27 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.databases.Database; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionDeleteAllOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyDeleteAllOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiCollectionDeleteAllOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionDeleteAllOpDispenser.class); - private final LongFunction opFunction; +public class DataApiCollectionLegacyDeleteAllOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyDeleteAllOpDispenser.class); + private final LongFunction opFunction; - public DataApiCollectionDeleteAllOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + public DataApiCollectionLegacyDeleteAllOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); - return new DataApiCollectionDeleteAllOp( + return new DataApiCollectionLegacyDeleteAllOp( db, db.getCollection(targetFunction.apply(l)) ); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteManyOpDispenser.java new file mode 100644 index 000000000..29d28a6a6 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteManyOpDispenser.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionDeleteManyOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyDeleteManyOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyDeleteManyOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyDeleteManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + + return new DataApiCollectionDeleteManyOp( + db, + db.getCollection(targetFunction.apply(l)), + filter + ); + }; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteOneOpDispenser.java new file mode 100644 index 000000000..58f49cd78 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyDeleteOneOpDispenser.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.collections.commands.options.CollectionDeleteOneOptions; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Sort; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionDeleteOneOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyDeleteOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyDeleteOneOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyDeleteOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionDeleteOneOptions options = getCollectionDeleteOneOptions(op, l); + + return new DataApiCollectionDeleteOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; + } + + private CollectionDeleteOneOptions getCollectionDeleteOneOptions(ParsedOp op, long l) { + CollectionDeleteOneOptions options = new CollectionDeleteOneOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindByIdOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindByIdOpDispenser.java similarity index 77% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindByIdOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindByIdOpDispenser.java index 58382780e..92e9262cc 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindByIdOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindByIdOpDispenser.java @@ -22,7 +22,7 @@ import com.datastax.astra.client.core.query.Projection; import com.datastax.astra.client.core.query.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindByIdOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyFindByIdOp; import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindOneOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; @@ -30,19 +30,19 @@ import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiCollectionFindByIdOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionFindByIdOpDispenser.class); - private final LongFunction opFunction; - public DataApiCollectionFindByIdOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { +public class DataApiCollectionLegacyFindByIdOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindByIdOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindByIdOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Object id = getIdFromOp(op, l); - return new DataApiCollectionFindByIdOp( + return new DataApiCollectionLegacyFindByIdOp( db, db.getCollection(targetFunction.apply(l)), id diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindDistinctOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindDistinctOpDispenser.java similarity index 75% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindDistinctOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindDistinctOpDispenser.java index cc20848e0..38e6701e0 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindDistinctOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindDistinctOpDispenser.java @@ -21,27 +21,27 @@ import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.core.query.Filter; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindDistinctOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyFindDistinctOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiCollectionFindDistinctOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionFindDistinctOpDispenser.class); - private final LongFunction opFunction; - public DataApiCollectionFindDistinctOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { +public class DataApiCollectionLegacyFindDistinctOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindDistinctOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindDistinctOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); - Filter filter = getFilterFromOp(op, l); + Filter filter = legacyGetFilterFromOp(op, l); Class targetClass = getTargetClass(op, l); - return new DataApiCollectionFindDistinctOp( + return new DataApiCollectionLegacyFindDistinctOp( db, db.getCollection(targetFunction.apply(l)), op.getAsRequiredFunction("fieldName", String.class).apply(l), diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndDeleteOpDispenser.java new file mode 100644 index 000000000..cf0c50ee5 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndDeleteOpDispenser.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneAndDeleteOptions; +import com.datastax.astra.client.core.query.Projection; +import com.datastax.astra.client.core.query.Sort; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindOneAndDeleteOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyFindOneAndDeleteOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindOneAndDeleteOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyFindOneAndDeleteOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionFindOneAndDeleteOptions options = getCollectionFindOneAndDeleteOptions(op, l); + + return new DataApiCollectionFindOneAndDeleteOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; + } + + private CollectionFindOneAndDeleteOptions getCollectionFindOneAndDeleteOptions(ParsedOp op, long l) { + CollectionFindOneAndDeleteOptions options = new CollectionFindOneAndDeleteOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndReplaceOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndReplaceOpDispenser.java new file mode 100644 index 000000000..d1869bf0c --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndReplaceOpDispenser.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import com.datastax.astra.client.collections.commands.ReturnDocument; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneAndReplaceOptions; +import com.datastax.astra.client.core.query.Projection; +import com.datastax.astra.client.core.query.Sort; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import com.datastax.astra.client.collections.definition.documents.Document; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindOneAndReplaceOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyFindOneAndReplaceOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindOneAndReplaceOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyFindOneAndReplaceOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionFindOneAndReplaceOptions options = getCollectionFindOneAndReplaceOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + + return new DataApiCollectionFindOneAndReplaceOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + docFunc.apply(l), + options + ); + }; + } + + private CollectionFindOneAndReplaceOptions getCollectionFindOneAndReplaceOptions(ParsedOp op, long l) { + CollectionFindOneAndReplaceOptions options = new CollectionFindOneAndReplaceOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + if (op.isDefined("returnDocument")) { + options = switch ((String) op.get("returnDocument", l)) { + case "after" -> options.returnDocument(ReturnDocument.AFTER); + case "before" -> options.returnDocument(ReturnDocument.BEFORE); + default -> throw new RuntimeException("Invalid returnDocument value: " + op.get("returnDocument", l)); + }; + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndUpdateOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndUpdateOpDispenser.java new file mode 100644 index 000000000..d3d2eb7ed --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneAndUpdateOpDispenser.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Sort; +import com.datastax.astra.client.collections.commands.Update; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneAndUpdateOptions; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyFindOneAndUpdateOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyFindOneAndUpdateOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindOneAndUpdateOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindOneAndUpdateOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + Update update = legacyGetUpdateFromOp(op, l); + CollectionFindOneAndUpdateOptions options = getCollectionLegacyFindOneAndUpdateOptions(op, l); + + return new DataApiCollectionLegacyFindOneAndUpdateOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + update, + options + ); + }; + } + + private CollectionFindOneAndUpdateOptions getCollectionLegacyFindOneAndUpdateOptions(ParsedOp op, long l) { + CollectionFindOneAndUpdateOptions options = new CollectionFindOneAndUpdateOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + Boolean upsert = getUpsertFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + if ( upsert != null ){ + options = options.upsert(upsert); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneOpDispenser.java new file mode 100644 index 000000000..18cbcda0c --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOneOpDispenser.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneOptions; +import com.datastax.astra.client.core.query.Projection; +import com.datastax.astra.client.core.query.Sort; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindOneOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyFindOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindOneOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionFindOneOptions options = getCollectionFindOneOptions(op, l); + return new DataApiCollectionFindOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; + } + + private CollectionFindOneOptions getCollectionFindOneOptions(ParsedOp op, long l) { + CollectionFindOneOptions options = new CollectionFindOneOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + Optional includeSimilarity = getIncludeSimilarityFromOp(op, l); + if (includeSimilarity.isPresent()) { + options.includeSimilarity(includeSimilarity.get()); + } else { + // defaulting to true (backward-compatibility) + options.includeSimilarity(true); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOpDispenser.java new file mode 100644 index 000000000..834040a38 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindOpDispenser.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.collections.commands.options.CollectionFindOptions; +import com.datastax.astra.client.core.query.Projection; +import com.datastax.astra.client.core.query.Sort; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyFindOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionFindOptions options = getCollectionFindOptions(op, l); + return new DataApiCollectionFindOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; + } + + private CollectionFindOptions getCollectionFindOptions(ParsedOp op, long l) { + CollectionFindOptions options = new CollectionFindOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + Optional> limitFunction = op.getAsOptionalFunction("limit", Integer.class); + if (limitFunction.isPresent()) { + options = options.limit(limitFunction.get().apply(l)); + } + Optional> skipFunction = op.getAsOptionalFunction("skip", Integer.class); + if (skipFunction.isPresent()) { + options = options.skip(skipFunction.get().apply(l)); + } + Optional> includeSimilarityFunction = op.getAsOptionalFunction("includeSimilarity", Boolean.class); + if (includeSimilarityFunction.isPresent()) { + options.includeSimilarity(includeSimilarityFunction.get().apply(l)); + } + Optional> pageStateFunction = op.getAsOptionalFunction("pageState", String.class); + if (pageStateFunction.isPresent()) { + options.pageState(pageStateFunction.get().apply(l)); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindVectorOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindVectorFilterOpDispenser.java similarity index 68% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindVectorOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindVectorFilterOpDispenser.java index 04bf31fd7..6e4945475 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindVectorOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindVectorFilterOpDispenser.java @@ -17,34 +17,36 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.databases.Database; -import com.datastax.astra.client.core.query.Sort; +import com.datastax.astra.client.core.query.Filter; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindVectorOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyFindVectorFilterOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiCollectionFindVectorOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionFindVectorOpDispenser.class); - private final LongFunction opFunction; - public DataApiCollectionFindVectorOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { +public class DataApiCollectionLegacyFindVectorFilterOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindVectorFilterOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindVectorFilterOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); float[] vector = getVectorValues(op, l); + Filter filter = legacyGetFilterFromOp(op, l); int limit = getLimit(op, l); - return new DataApiCollectionFindVectorOp( + return new DataApiCollectionLegacyFindVectorFilterOp( db, db.getCollection(targetFunction.apply(l)), vector, - limit + limit, + filter ); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindVectorFilterOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindVectorOpDispenser.java similarity index 75% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindVectorFilterOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindVectorOpDispenser.java index 6a3cd1aeb..c74151b01 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionFindVectorFilterOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyFindVectorOpDispenser.java @@ -17,36 +17,34 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.databases.Database; -import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionFindVectorFilterOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyFindVectorOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiCollectionFindVectorFilterOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionFindVectorFilterOpDispenser.class); - private final LongFunction opFunction; - public DataApiCollectionFindVectorFilterOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { +public class DataApiCollectionLegacyFindVectorOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyFindVectorOpDispenser.class); + private final LongFunction opFunction; + public DataApiCollectionLegacyFindVectorOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); float[] vector = getVectorValues(op, l); - Filter filter = getFilterFromOp(op, l); int limit = getLimit(op, l); - return new DataApiCollectionFindVectorFilterOp( + return new DataApiCollectionLegacyFindVectorOp( db, db.getCollection(targetFunction.apply(l)), vector, - limit, - filter + limit ); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyInsertManyOpDispenser.java new file mode 100644 index 000000000..a5b2a2c4f --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyInsertManyOpDispenser.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.collections.commands.options.CollectionInsertManyOptions; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionInsertManyOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyInsertManyOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyInsertManyOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyInsertManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + List documents = new ArrayList<>(); + op.getAsRequiredFunction("documents", List.class).apply(l).forEach(o -> documents.add(Document.parse(o.toString()))); + return new DataApiCollectionInsertManyOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + documents, + getCollectionInsertManyOptions(op, l) + ); + }; + } + + private CollectionInsertManyOptions getCollectionInsertManyOptions(ParsedOp op, long l) { + CollectionInsertManyOptions options = new CollectionInsertManyOptions(); + Optional> optionsFunction = op.getAsOptionalFunction("options", Map.class); + if (optionsFunction.isPresent()) { + Map optionFields = optionsFunction.get().apply(l); + for(Map.Entry entry: optionFields.entrySet()) { + switch(entry.getKey()) { + case "chunkSize"-> + options = options.chunkSize(Integer.parseInt(entry.getValue())); + case "concurrency" -> + options = options.concurrency(Integer.parseInt(entry.getValue())); + case "ordered" -> + options = options.ordered(Boolean.parseBoolean(entry.getValue())); + case "timeout" -> + options = options.timeout(Integer.parseInt(entry.getValue())); + } + } + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneVectorOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyInsertOneVectorOpDispenser.java similarity index 68% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneVectorOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyInsertOneVectorOpDispenser.java index b157f921c..1278ce106 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionInsertOneVectorOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyInsertOneVectorOpDispenser.java @@ -19,8 +19,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.collections.definition.documents.Document; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionInsertOneOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionInsertOneVectorOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyInsertOneVectorOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,25 +27,24 @@ import org.apache.logging.log4j.Logger; import java.util.Map; import java.util.function.LongFunction; -public class DataApiCollectionInsertOneVectorOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionInsertOneVectorOpDispenser.class); - private final LongFunction opFunction; +public class DataApiCollectionLegacyInsertOneVectorOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyInsertOneVectorOpDispenser.class); + private final LongFunction opFunction; - public DataApiCollectionInsertOneVectorOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + public DataApiCollectionLegacyInsertOneVectorOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); - LongFunction vectorF= op.getAsRequiredFunction("vector", float[].class); return (l) -> { - return new DataApiCollectionInsertOneVectorOp( + return new DataApiCollectionLegacyInsertOneVectorOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), docFunc.apply(l), - vectorF.apply(l) + getVectorValues(op, l) ); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyReplaceOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyReplaceOneOpDispenser.java new file mode 100644 index 000000000..46a2fc6df --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyReplaceOneOpDispenser.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Projection; +import com.datastax.astra.client.core.query.Sort; +import com.datastax.astra.client.collections.commands.ReturnDocument; +import com.datastax.astra.client.collections.commands.options.CollectionReplaceOneOptions; +import com.datastax.astra.client.collections.definition.documents.Document; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionLegacyReplaceOneOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyReplaceOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyReplaceOneOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyReplaceOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionReplaceOneOptions options = getCollectionReplaceOneOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + + return new DataApiCollectionLegacyReplaceOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + docFunc.apply(l), + options + ); + }; + } + + private CollectionReplaceOneOptions getCollectionReplaceOneOptions(ParsedOp op, long l) { + CollectionReplaceOneOptions options = new CollectionReplaceOneOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + if (op.isDefined("returnDocument")) { + options = switch ((String) op.get("returnDocument", l)) { + case "after" -> options.returnDocument(ReturnDocument.AFTER); + case "before" -> options.returnDocument(ReturnDocument.BEFORE); + default -> throw new RuntimeException("Invalid returnDocument value: " + op.get("returnDocument", l)); + }; + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionReplaceOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyUpdateManyOpDispenser.java similarity index 66% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionReplaceOneOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyUpdateManyOpDispenser.java index 8e46ab504..9a96daa8e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionReplaceOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyUpdateManyOpDispenser.java @@ -18,11 +18,11 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.core.query.Filter; -import com.datastax.astra.client.collections.commands.options.CollectionReplaceOneOptions; -import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.collections.commands.Update; +import com.datastax.astra.client.collections.commands.options.CollectionUpdateManyOptions; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionReplaceOneOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionUpdateManyOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,40 +31,39 @@ import java.util.Map; import java.util.Optional; import java.util.function.LongFunction; -public class DataApiCollectionReplaceOneOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCollectionReplaceOneOpDispenser.class); - private final LongFunction opFunction; +public class DataApiCollectionLegacyUpdateManyOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyUpdateManyOpDispenser.class); + private final LongFunction opFunction; - public DataApiCollectionReplaceOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + public DataApiCollectionLegacyUpdateManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); - Filter filter = getFilterFromOp(op, l); - CollectionReplaceOneOptions options = getCollectionReplaceOneOptions(op, l); - LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); - LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionUpdateManyOptions options = getCollectionUpdateManyOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("updates", Map.class); - return new DataApiCollectionReplaceOneOp( + return new DataApiCollectionUpdateManyOp( db, db.getCollection(targetFunction.apply(l)), filter, - docFunc.apply(l), + new Update(docMapFunc.apply(l)), options ); }; } - private CollectionReplaceOneOptions getCollectionReplaceOneOptions(ParsedOp op, long l) { - CollectionReplaceOneOptions options = new CollectionReplaceOneOptions(); - + private CollectionUpdateManyOptions getCollectionUpdateManyOptions(ParsedOp op, long l) { + CollectionUpdateManyOptions options = new CollectionUpdateManyOptions(); Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); if (upsertFunction.isPresent()) { options = options.upsert(upsertFunction.get().apply(l)); } + return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyUpdateOneOpDispenser.java new file mode 100644 index 000000000..348f5c38b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionLegacyUpdateOneOpDispenser.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.collections.commands.options.CollectionUpdateOneOptions; +import com.datastax.astra.client.core.query.Sort; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.collections.commands.Update; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCollectionUpdateOneOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiCollectionLegacyUpdateOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCollectionLegacyUpdateOneOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCollectionLegacyUpdateOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = legacyGetFilterFromOp(op, l); + CollectionUpdateOneOptions options = getCollectionUpdateOneOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("update", Map.class); + + return new DataApiCollectionUpdateOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + new Update(docMapFunc.apply(l)), + options + ); + }; + } + + private CollectionUpdateOneOptions getCollectionUpdateOneOptions(ParsedOp op, long l) { + CollectionUpdateOneOptions options = new CollectionUpdateOneOptions(); + Sort[] sorts = legacyGetSortFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); + } + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateManyOpDispenser.java index 7a920d7c4..f37860095 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateManyOpDispenser.java @@ -27,8 +27,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class DataApiCollectionUpdateManyOpDispenser extends DataApiOpDispenser { @@ -45,13 +43,13 @@ public class DataApiCollectionUpdateManyOpDispenser extends DataApiOpDispenser { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); CollectionUpdateManyOptions options = getCollectionUpdateManyOptions(op, l); - LongFunction docMapFunc = op.getAsRequiredFunction("updates", Map.class); + Update update = getUpdateFromOp(op, l); return new DataApiCollectionUpdateManyOp( db, db.getCollection(targetFunction.apply(l)), filter, - new Update(docMapFunc.apply(l)), + update, options ); }; @@ -59,11 +57,10 @@ public class DataApiCollectionUpdateManyOpDispenser extends DataApiOpDispenser { private CollectionUpdateManyOptions getCollectionUpdateManyOptions(ParsedOp op, long l) { CollectionUpdateManyOptions options = new CollectionUpdateManyOptions(); - Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); - if (upsertFunction.isPresent()) { - options = options.upsert(upsertFunction.get().apply(l)); + Boolean upsert = getUpsertFromOp(op, l); + if (upsert != null) { + options = options.upsert(upsert); } - return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateOneOpDispenser.java index d7c26b850..0b3cb5f4b 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCollectionUpdateOneOpDispenser.java @@ -46,13 +46,13 @@ public class DataApiCollectionUpdateOneOpDispenser extends DataApiOpDispenser { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); CollectionUpdateOneOptions options = getCollectionUpdateOneOptions(op, l); - LongFunction docMapFunc = op.getAsRequiredFunction("update", Map.class); + Update update = getUpdateFromOp(op, l); return new DataApiCollectionUpdateOneOp( db, db.getCollection(targetFunction.apply(l)), filter, - new Update(docMapFunc.apply(l)), + update, options ); }; @@ -60,23 +60,17 @@ public class DataApiCollectionUpdateOneOpDispenser extends DataApiOpDispenser { private CollectionUpdateOneOptions getCollectionUpdateOneOptions(ParsedOp op, long l) { CollectionUpdateOneOptions options = new CollectionUpdateOneOptions(); - Sort sort = getSortFromOp(op, l); - float[] vector = getVectorFromOp(op, l); - - Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); - if (upsertFunction.isPresent()) { - options = options.upsert(upsertFunction.get().apply(l)); + Sort[] sorts = getSortFromOp(op, l); + Boolean upsert = getUpsertFromOp(op, l); + if (sorts != null) { + options = options.sort(sorts); } - if (sort != null) { - options = (vector != null) ? options.sort(Sort.vector(vector), sort) : options.sort(sort); + if ( upsert != null ){ + options = options.upsert(upsert); } return options; } - private float[] getVectorFromOp(ParsedOp op, long l) { - return getVectorValues(op.get("vector", l)); - } - @Override public DataApiBaseOp getOp(long cycle) { return opFunction.apply(cycle); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDbCreateCollectionOpDispenser.java similarity index 68% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDbCreateCollectionOpDispenser.java index 92b1f38d6..1e61fafc3 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDbCreateCollectionOpDispenser.java @@ -18,33 +18,32 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCreateCollectionOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDbCreateCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Optional; import java.util.function.LongFunction; -public class DataApiCreateCollectionOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCreateCollectionOpDispenser.class); - private final LongFunction opFunction; +public class DataApiDbCreateCollectionOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDbCreateCollectionOpDispenser.class); + private final LongFunction opFunction; - public DataApiCreateCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + public DataApiDbCreateCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { return (l) -> { - DataApiCreateCollectionOp dataApiCreateCollectionOp = - new DataApiCreateCollectionOp( + DataApiDbCreateCollectionOp dataApiDbCreateCollectionOp = + new DataApiDbCreateCollectionOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), this.getCollectionDefinitionFromOp(op, l) ); - return dataApiCreateCollectionOp; + return dataApiDbCreateCollectionOp; }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDbDeleteCollectionOpDispenser.java similarity index 70% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDbDeleteCollectionOpDispenser.java index 15caa8afc..701197d42 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDbDeleteCollectionOpDispenser.java @@ -18,23 +18,23 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiDropCollectionOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiLegacyDeleteCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiDropCollectionOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiDropCollectionOpDispenser.class); - private final LongFunction opFunction; - public DataApiDropCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { +public class DataApiDbDeleteCollectionOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDbDeleteCollectionOpDispenser.class); + private final LongFunction opFunction; + public DataApiDbDeleteCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { - return (l) -> new DataApiDropCollectionOp( + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiLegacyDeleteCollectionOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l) ); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyCreateCollectionOpDispenser.java new file mode 100644 index 000000000..f3bbab85a --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyCreateCollectionOpDispenser.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiLegacyCreateCollectionOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiLegacyCreateCollectionOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiLegacyCreateCollectionOpDispenser.class); + private final LongFunction opFunction; + + public DataApiLegacyCreateCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + DataApiLegacyCreateCollectionOp dataApiCreateCollectionOp = + new DataApiLegacyCreateCollectionOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + this.legacyGetCollectionDefinitionFromOp(op, l) + ); + + return dataApiCreateCollectionOp; + }; + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } + + +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyCreateCollectionWithClassOpDispenser.java similarity index 71% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyCreateCollectionWithClassOpDispenser.java index 76aca2e7a..0f5f3a464 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyCreateCollectionWithClassOpDispenser.java @@ -18,27 +18,27 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapter.dataapi.ops.DataApiCreateCollectionWithClassOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiLegacyCreateCollectionWithClassOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class DataApiCreateCollectionWithClassOpDispenser extends DataApiOpDispenser { - private static final Logger logger = LogManager.getLogger(DataApiCreateCollectionWithClassOpDispenser.class); - private final LongFunction opFunction; +public class DataApiLegacyCreateCollectionWithClassOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiLegacyCreateCollectionWithClassOpDispenser.class); + private final LongFunction opFunction; - public DataApiCreateCollectionWithClassOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + public DataApiLegacyCreateCollectionWithClassOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { - return (l) -> new DataApiCreateCollectionWithClassOp( + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiLegacyCreateCollectionWithClassOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), - this.getCollectionDefinitionFromOp(op, l), + this.legacyGetCollectionDefinitionFromOp(op, l), getCreateClass(op, l) ); } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyDeleteCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyDeleteCollectionOpDispenser.java new file mode 100644 index 000000000..65c933f32 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiLegacyDeleteCollectionOpDispenser.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiLegacyDeleteCollectionOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiLegacyDeleteCollectionOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiLegacyDeleteCollectionOpDispenser.class); + private final LongFunction opFunction; + public DataApiLegacyDeleteCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiLegacyDeleteCollectionOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l) + ); + } + + @Override + public DataApiBaseOp getOp(long cycle) { + return opFunction.apply(cycle); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index 1f6f32223..171853a12 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -20,12 +20,18 @@ import com.datastax.astra.client.core.query.Filter; import com.datastax.astra.client.core.query.Filters; import com.datastax.astra.client.collections.definition.CollectionDefinition; import com.datastax.astra.client.collections.definition.CollectionDefaultIdTypes; +import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.collections.commands.ReturnDocument; import com.datastax.astra.client.collections.commands.Update; import com.datastax.astra.client.collections.commands.Updates; +import com.datastax.astra.client.core.hybrid.Hybrid; import com.datastax.astra.client.core.query.Sort; +import com.datastax.astra.client.core.rerank.RerankServiceOptions; +import com.datastax.astra.client.core.vector.DataAPIVector; import com.datastax.astra.client.core.vector.SimilarityMetric; import com.datastax.astra.client.core.query.Projection; import io.nosqlbench.adapter.dataapi.DataApiSpace; +import io.nosqlbench.nb.api.errors.OpConfigError; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; @@ -33,6 +39,8 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.*; import java.util.function.LongFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; public abstract class DataApiOpDispenser extends BaseOpDispenser { protected final LongFunction targetFunction; @@ -45,108 +53,101 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser sorts = null; + + long sortKeyCount = Stream.of("sort", "vector", "vectorize").filter(op::isDefined).count(); + if (sortKeyCount > 1) { + throw new OpConfigError( + "Can sort by only one of: 'sort' (regular asc/desc), 'vector', 'vectorize' in an op." + ); + } + Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); if (sortFunction.isPresent()) { + @SuppressWarnings("unchecked") Map sortFields = sortFunction.get().apply(l); - String sortOrder = sortFields.get("type").toString(); - String sortField = sortFields.get("field").toString(); - switch(sortOrder) { - case "asc" -> sort = Sort.ascending(sortField); - case "desc" -> sort = Sort.descending(sortField); + + sorts = sortFields.entrySet().stream() + .map(e -> { + String field = e.getKey(); + String sortOrder = String.valueOf(e.getValue()).trim(); + if (sortOrder.equalsIgnoreCase("asc") || sortOrder.equalsIgnoreCase("ascending")) { + return Sort.ascending(field); + } else if (sortOrder.equalsIgnoreCase("desc") || sortOrder.equalsIgnoreCase("descending")) { + return Sort.descending(field); + } else { + throw new OpConfigError( + "Invalid sort order '" + sortOrder + "' for field '" + field + + "'; expected 'asc' or 'desc' (case-insensitive)." + ); + } + }) + .collect(Collectors.toList()); + } + + if (op.isDefined("vector")) { + float[] vector = getVectorValues(op, l); + if (vector != null) { + // TODO use DataAPIVector as soon as the client allows here: + sorts = List.of(Sort.vector(vector)); } } - return sort; + + if (op.isDefined("vectorize")) { + Optional> vzeFunction = op.getAsOptionalFunction("vectorize", String.class); + if (vzeFunction.isPresent()){ + String vectorize = vzeFunction.get().apply(l); + sorts = List.of(Sort.vectorize(vectorize)); + } + } + + if (sorts != null) { + return sorts.toArray(new Sort[0]); + } + return null; } protected Filter getFilterFromOp(ParsedOp op, long l) { - // TODO: Clarify 'filter' vs 'filters' or whether to support both uniformly - Filter filter = null; - Optional> filterFunction = op.getAsOptionalFunction("filters", List.class) - .or(() -> op.getAsOptionalFunction("filter",List.class)); - - if (filterFunction.isPresent()) { - List> filters = filterFunction.get().apply(l); - List andFilterList = new ArrayList<>(); - List orFilterList = new ArrayList<>(); - for (Map filterFields : filters) { - switch ((String)filterFields.get("conjunction")) { - case "and" -> - addOperatorFilter(andFilterList, filterFields.get("operator").toString(), filterFields.get("field").toString(), filterFields.get("value")); - case "or" -> - addOperatorFilter(orFilterList, filterFields.get("operator").toString(), filterFields.get("field").toString(), filterFields.get("value")); - default -> logger.error(() -> "Conjunction " + filterFields.get("conjunction") + " not supported"); - } - } - if (!andFilterList.isEmpty()) - filter = Filters.and(andFilterList.toArray(new Filter[0])); - if (!orFilterList.isEmpty()) - filter = Filters.or(orFilterList.toArray(new Filter[0])); + Map filterMap = getFreeFormFromOp(op, l, "filter", false); + if (filterMap != null) { + return new Filter(filterMap); } - return filter; + return null; } - protected void addOperatorFilter(List filtersList, String operator, String fieldName, Object fieldValue) { - switch (operator) { - case "all" -> - filtersList.add(Filters.all(fieldName, fieldValue)); - case "eq" -> - filtersList.add(Filters.eq(fieldName, fieldValue)); - case "exists" -> { - if (fieldValue != null) { - logger.warn(() -> "'exists' operator does not support value field"); - } - filtersList.add(Filters.exists(fieldName)); - } - case "gt" -> - filtersList.add(Filters.gt(fieldName, (long) fieldValue)); - case "gte" -> - filtersList.add(Filters.gte(fieldName, (long) fieldValue)); - case "hasSize" -> - filtersList.add(Filters.hasSize(fieldName, (int) fieldValue)); - case "in" -> - filtersList.add(Filters.in(fieldName, fieldValue)); - case "lt" -> - filtersList.add(Filters.lt(fieldName, (long) fieldValue)); - case "lte" -> - filtersList.add(Filters.lte(fieldName, (long) fieldValue)); - case "ne" -> - filtersList.add(Filters.ne(fieldName, fieldValue)); - case "nin" -> - filtersList.add(Filters.nin(fieldName, fieldValue)); - default -> logger.error(() -> "Operation '" + operator + "' not supported"); - } + protected Document getReplacementFromOp(ParsedOp op, long l) { + Map docMap = getFreeFormFromOp(op, l, "replacement", true); + return new Document(docMap); } - protected Update getUpdates(ParsedOp op, long l) { - Update update = new Update(); - Optional> updatesFunction = op.getAsOptionalFunction("updates", Map.class); - if (updatesFunction.isPresent()) { - Map updates = updatesFunction.get().apply(l); - for (Map.Entry entry : updates.entrySet()) { - if (entry.getKey().equalsIgnoreCase("update")) { - Map updateFields = (Map) entry.getValue(); - switch (updateFields.get("operation").toString()) { - case "set" -> - update = Updates.set(updateFields.get("field").toString(), updateFields.get("value")); - case "inc" -> - update = Updates.inc(updateFields.get("field").toString(), (double) updateFields.get("value")); - case "unset" -> update = Updates.unset(updateFields.get("field").toString()); - case "addToSet" -> - update = Updates.addToSet(updateFields.get("field").toString(), updateFields.get("value")); - case "min" -> - update = Updates.min(updateFields.get("field").toString(), (double) updateFields.get("value")); - case "rename" -> - update = Updates.rename(updateFields.get("field").toString(), updateFields.get("value").toString()); - default -> logger.error(() -> "Operation " + updateFields.get("operation") + " not supported"); - } - } else { - logger.error(() -> "Filter " + entry.getKey() + " not supported"); - } - } + protected Document getDocumentFromRawMap(Map docMap) { + Document doc = new Document(docMap); + // if docMap has a '$vector' key, invoke: + if (docMap.containsKey("$vector")) { + doc.vector(new DataAPIVector(getVectorValues(docMap.get("$vector")))); } - return update; + return doc; + } + + protected Document getDocumentFromOp(ParsedOp op, long l) { + Map docMap = getFreeFormFromOp(op, l, "document", true); + return getDocumentFromRawMap(docMap); + } + + protected List getDocumentsFromOp(ParsedOp op, long l) { + List> docMapList = getFreeFormListFromOp(op, l, "documents", true); + return docMapList.stream().map((docMap) -> getDocumentFromRawMap(docMap)).toList(); + } + + protected Update getUpdateFromOp(ParsedOp op, long l) { + Map updateMap = getFreeFormFromOp(op, l, "update", true); + return new Update(updateMap); + } + + protected Map getCollectionDefinitionFromOp(ParsedOp op, long l) { + Map definition = getFreeFormFromOp(op, l, "definition", true); + return definition; } protected float[] getVectorValues(ParsedOp op, long l) { @@ -167,6 +168,8 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser vectorValuesList = (List) rawVectorValues; vectorValues = new float[vectorValuesList.size()]; for (int i = 0; i < vectorValuesList.size(); i++) { @@ -184,30 +188,418 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> projectionFunction = op.getAsOptionalFunction("projection", Map.class); - if (projectionFunction.isPresent()) { - Map> projectionFields = projectionFunction.get().apply(l); - for (Map.Entry> field : projectionFields.entrySet()) { - List includeFields = field.getValue(); - StringBuffer sb = new StringBuffer(); - for (String includeField : includeFields) { - sb.append(includeField).append(","); - } - sb.deleteCharAt(sb.length() - 1); + if (projectionFunction.isEmpty()) { + return null; + } + @SuppressWarnings("unchecked") + Map> projectionFields = projectionFunction.get().apply(l); + return projectionFields.entrySet().stream() + .flatMap(field -> { + String[] arr = field.getValue().toArray(new String[0]); if (field.getKey().equalsIgnoreCase("include")) { - projection = Projection.include(sb.toString()); + return Arrays.stream(Projection.include(arr)); } else if (field.getKey().equalsIgnoreCase("exclude")) { - projection = Projection.exclude(sb.toString()); + return Arrays.stream(Projection.exclude(arr)); } else { - logger.error("Projection " + field + " not supported"); + logger.error("Projection directive '" + field.getKey() + "' not supported"); + return Stream.empty(); + } + }).toArray(Projection[]::new); + } + + protected Boolean getUpsertFromOp(ParsedOp op, long l) { + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + LongFunction uf = upsertFunction.get(); + return uf.apply(l); + } + return null; + } + + protected Integer getChunkSizeFromOp(ParsedOp op, long l) { + Optional> csFunction = op.getAsOptionalFunction("chunk_size", Integer.class); + if (csFunction.isPresent()) { + LongFunction cs = csFunction.get(); + return cs.apply(l); + } + return null; + } + + protected Boolean getOrderedFromOp(ParsedOp op, long l) { + Optional> orderedFunction = op.getAsOptionalFunction("ordered", Boolean.class); + if (orderedFunction.isPresent()) { + LongFunction of = orderedFunction.get(); + return of.apply(l); + } + return null; + } + + protected ReturnDocument getReturnDocumentFromOp(ParsedOp op, long l) { + Optional> rdf = op.getAsOptionalFunction("return_document", String.class); + if (rdf.isPresent()) { + String rdValue = rdf.get().apply(l); + return switch (rdValue) { + case "after" -> ReturnDocument.AFTER; + case "before" -> ReturnDocument.BEFORE; + default -> throw new RuntimeException("Invalid returnDocument value: " + op.get("return_document", l)); + }; + } + return null; + } + + protected Optional getIncludeSimilarityFromOp(ParsedOp op, long l) { + Optional> includeSimFunction = op.getAsOptionalFunction("include_similarity", Boolean.class); + if (includeSimFunction.isPresent()) { + LongFunction sf = includeSimFunction.get(); + return Optional.of(sf.apply(l)); + } + return Optional.empty(); + } + + protected Optional getIncludeSortVectorFromOp(ParsedOp op, long l) { + Optional> includeSvFunction = op.getAsOptionalFunction("include_sort_vector", Boolean.class); + if (includeSvFunction.isPresent()) { + LongFunction sv = includeSvFunction.get(); + return Optional.of(sv.apply(l)); + } + return Optional.empty(); + } + + protected Optional getLimitFromOp(ParsedOp op, long l) { + Optional> limitFunction = op.getAsOptionalFunction("limit", Integer.class); + if (limitFunction.isPresent()) { + LongFunction lf = limitFunction.get(); + return Optional.of(lf.apply(l)); + } + return Optional.empty(); + } + + protected Optional getSkipFromOp(ParsedOp op, long l) { + Optional> skipFunction = op.getAsOptionalFunction("skip", Integer.class); + if (skipFunction.isPresent()) { + LongFunction sf = skipFunction.get(); + return Optional.of(sf.apply(l)); + } + return Optional.empty(); + } + + @SuppressWarnings("unchecked") + protected Map getFreeFormFromOp(ParsedOp op, long l, String fieldName, Boolean required) { + Optional> ffMapFunc = op.getAsOptionalFunction(fieldName, Map.class); + if (ffMapFunc.isPresent()) { + LongFunction dmf = ffMapFunc.get(); + return dmf.apply(l); + } else { + if (required) { + throw new OpConfigError( + "Required field '" + fieldName + "' not supplied." + ); + } + return null; + } + } + + @SuppressWarnings("unchecked") + protected List> getFreeFormListFromOp(ParsedOp op, long l, String fieldName, Boolean required) { + Optional> ffMapFunc = op.getAsOptionalFunction(fieldName, List.class); + if (ffMapFunc.isPresent()) { + LongFunction dmf = ffMapFunc.get(); + return dmf.apply(l); + } else { + if (required) { + throw new OpConfigError( + "Required field '" + fieldName + "' not supplied." + ); + } + return null; + } + } + + protected Optional getIncludeScoresFromOp(ParsedOp op, long l) { + Optional> includeScFunction = op.getAsOptionalFunction("include_scores", Boolean.class); + if (includeScFunction.isPresent()) { + LongFunction isf = includeScFunction.get(); + return Optional.of(isf.apply(l)); + } + return Optional.empty(); + } + + protected Optional getRerankQueryFromOp(ParsedOp op, long l) { + Optional> rqf = op.getAsOptionalFunction("rerank_query", String.class); + if (rqf.isPresent()) { + String rqValue = rqf.get().apply(l); + return Optional.of(rqValue); + } + return Optional.empty(); + } + + protected Optional getRerankOnFromOp(ParsedOp op, long l) { + Optional> rof = op.getAsOptionalFunction("rerank_on", String.class); + if (rof.isPresent()) { + String roValue = rof.get().apply(l); + return Optional.of(roValue); + } + return Optional.empty(); + } + + @SuppressWarnings("unchecked") + protected Optional getHybridLimitsFromOp(ParsedOp op, long l) { + Optional> hlFunc = op.getAsOptionalFunction("hybrid_limits", Object.class); + if (hlFunc.isPresent()) { + Object hlValue = hlFunc.get().apply(l); + if (hlValue instanceof Integer || hlValue instanceof Long) { + return Optional.of(hlValue); + } else if (hlValue instanceof Map rawMap) { + return Optional.of((Map) rawMap); + } else { + throw new OpConfigError( + "'hybrid_limits' must be an integer or a map, got: " + hlValue.getClass().getSimpleName() + ); + } + } + return Optional.empty(); + } + + protected Optional getRerankServiceFromOp(ParsedOp op, long l) { + Map rerankService = getFreeFormFromOp(op, l, "rerank", false); + if (rerankService != null) { + RerankServiceOptions rsOptions = new RerankServiceOptions(); + if (rerankService.containsKey("provider")) { + rsOptions = rsOptions.provider((String) rerankService.get("provider")); + } + if (rerankService.containsKey("modelName")) { + rsOptions = rsOptions.modelName((String) rerankService.get("modelName")); + } + if (rerankService.containsKey("authentication")) { + rsOptions = rsOptions.authentication((Map) rerankService.get("authentication")); + } + if (rerankService.containsKey("parameters")) { + rsOptions = rsOptions.parameters((Map) rerankService.get("parameters")); + } + return Optional.of(rsOptions); + } + return Optional.empty(); + } + + protected Sort getFindAndRerankSortFromOp(ParsedOp op, long l) { + Map docMap = getFreeFormFromOp(op, l, "sort", true); + + if (docMap.size() != 1 || !docMap.containsKey("$hybrid")) { + throw new OpConfigError("'sort' must contain exactly one key: '$hybrid'"); + } + + Object hybridValue = docMap.get("$hybrid"); + + if (hybridValue instanceof String hybridString) { + return Sort.hybrid(hybridString); + } + + if (hybridValue instanceof Map rawMap) { + @SuppressWarnings("unchecked") + Map hybMap = (Map) rawMap; + + boolean hasVectorize = hybMap.containsKey("$vectorize"); + boolean hasVector = hybMap.containsKey("$vector"); + if (!hasVectorize && !hasVector) { + throw new OpConfigError( + "'$hybrid' sub-object must contain at least one of '$vectorize' or '$vector'" + ); + } + + Set allowedKeys = Set.of("$vectorize", "$vector", "$lexical"); + for (String key : hybMap.keySet()) { + if (!allowedKeys.contains(key)) { + throw new OpConfigError("Unexpected key in '$hybrid' sub-object: '" + key + "'"); + } + } + + Hybrid hyb = new Hybrid(); + if (hasVectorize) { + hyb = hyb.vectorize((String) hybMap.get("$vectorize")); + } + if (hasVector) { + hyb = hyb.vector(new DataAPIVector(getVectorValues(hybMap.get("$vector")))); + } + if (hybMap.containsKey("$lexical")) { + hyb = hyb.lexical((String) hybMap.get("$lexical")); + } + return Sort.hybrid(hyb); + } + + throw new OpConfigError( + "'$hybrid' value must be a string or a sub-object, got: " + hybridValue.getClass().getSimpleName() + ); + } + + /* LEGACY OP DISPENSER UTILS START HERE */ + + /* + updates: + update: + operation: inc + field: "incd field" + value: 500 + */ + protected Update legacyGetUpdateFromOp(ParsedOp op, long l) { + Update update = new Update(); + Optional> updatesFunction = op.getAsOptionalFunction("updates", Map.class); + if (updatesFunction.isPresent()) { + @SuppressWarnings("unchecked") + Map updates = updatesFunction.get().apply(l); + for (Map.Entry entry : updates.entrySet()) { + if (entry.getKey().equalsIgnoreCase("update")) { + @SuppressWarnings("unchecked") + Map updateFields = (Map) entry.getValue(); + switch (updateFields.get("operation").toString()) { + case "set" -> + update = Updates.set(updateFields.get("field").toString(), updateFields.get("value")); + case "inc" -> + update = Updates.inc(updateFields.get("field").toString(), ((Number) updateFields.get("value")).doubleValue()); + case "unset" -> update = Updates.unset(updateFields.get("field").toString()); + case "addToSet" -> + update = Updates.addToSet(updateFields.get("field").toString(), updateFields.get("value")); + case "min" -> + update = Updates.min(updateFields.get("field").toString(), ((Number) updateFields.get("value")).doubleValue()); + case "rename" -> + update = Updates.rename(updateFields.get("field").toString(), updateFields.get("value").toString()); + default -> logger.error(() -> "Operation " + updateFields.get("operation") + " not supported"); + } + } else { + logger.error(() -> "Unsupported entry under 'updates': '" + entry.getKey() + "'"); } } } - return projection; + return update; } - protected CollectionDefinition getCollectionDefinitionFromOp(ParsedOp op, long l) { + protected void legacyAddOperatorFilter(List filtersList, String operator, String fieldName, Object fieldValue) { + switch (operator) { + case "all" -> + filtersList.add(Filters.all(fieldName, fieldValue)); + case "eq" -> + filtersList.add(Filters.eq(fieldName, fieldValue)); + case "exists" -> { + if (fieldValue != null) { + logger.warn(() -> "'exists' operator does not support value field"); + } + filtersList.add(Filters.exists(fieldName)); + } + case "gt" -> + filtersList.add(Filters.gt(fieldName, ((Number) fieldValue).longValue())); + case "gte" -> + filtersList.add(Filters.gte(fieldName, ((Number) fieldValue).longValue())); + case "hasSize" -> + filtersList.add(Filters.hasSize(fieldName, ((Number) fieldValue).intValue())); + case "in" -> + filtersList.add(Filters.in(fieldName, fieldValue)); + case "lt" -> + filtersList.add(Filters.lt(fieldName, ((Number) fieldValue).longValue())); + case "lte" -> + filtersList.add(Filters.lte(fieldName, ((Number) fieldValue).longValue())); + case "ne" -> + filtersList.add(Filters.ne(fieldName, fieldValue)); + case "nin" -> + filtersList.add(Filters.nin(fieldName, fieldValue)); + default -> logger.error(() -> "Operation '" + operator + "' not supported"); + } + } + + /* + filters: + - conjunction: "and" + operator: "eq" + field: "field1" + value: 74 + - conjunction: "and" + operator: "lt" + field: "field2" + value: 111 + */ + protected Filter legacyGetFilterFromOp(ParsedOp op, long l) { + Filter filter = null; + Optional> filterFunction = op.getAsOptionalFunction("filters", List.class) + .or(() -> op.getAsOptionalFunction("filter",List.class)); + + if (filterFunction.isPresent()) { + @SuppressWarnings("unchecked") + List> filters = filterFunction.get().apply(l); + List andFilterList = new ArrayList<>(); + List orFilterList = new ArrayList<>(); + for (Map filterFields : filters) { + switch ((String)filterFields.get("conjunction")) { + case "and" -> + legacyAddOperatorFilter(andFilterList, filterFields.get("operator").toString(), filterFields.get("field").toString(), filterFields.get("value")); + case "or" -> + legacyAddOperatorFilter(orFilterList, filterFields.get("operator").toString(), filterFields.get("field").toString(), filterFields.get("value")); + default -> logger.error(() -> "Conjunction " + filterFields.get("conjunction") + " not supported"); + } + } + if (!andFilterList.isEmpty() && !orFilterList.isEmpty()) { + throw new OpConfigError( + "filters list mixes 'and' and 'or' conjunctions, which is not supported; " + + "use only one conjunction type per filters list" + ); + } + if (!andFilterList.isEmpty()) + filter = Filters.and(andFilterList.toArray(new Filter[0])); + if (!orFilterList.isEmpty()) + filter = Filters.or(orFilterList.toArray(new Filter[0])); + } + return filter; + } + + /* + sort: + field: the_field + type: desc + */ + protected Sort[] legacyGetSortFromOp(ParsedOp op, long l) { + List sorts = null; + + long sortKeyCount = Stream.of("sort", "vector", "vectorize").filter(op::isDefined).count(); + if (sortKeyCount > 1) { + throw new OpConfigError( + "Can sort by only one of: 'sort' (regular asc/desc), 'vector', 'vectorize' in an op." + ); + } + + Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); + if (sortFunction.isPresent()) { + @SuppressWarnings("unchecked") + Map sortFields = sortFunction.get().apply(l); + String sortOrder = sortFields.get("type").toString(); + String sortField = sortFields.get("field").toString(); + switch(sortOrder) { + case "asc" -> sorts = List.of(Sort.ascending(sortField)); + case "desc" -> sorts = List.of(Sort.descending(sortField)); + } + } + + if (op.isDefined("vector")) { + float[] vector = getVectorValues(op, l); + if (vector != null) { + // TODO use DataAPIVector as soon as the client allows here: + sorts = List.of(Sort.vector(vector)); + } + } + + if (op.isDefined("vectorize")) { + Optional> vzeFunction = op.getAsOptionalFunction("vectorize", String.class); + if (vzeFunction.isPresent()){ + String vectorize = vzeFunction.get().apply(l); + sorts = List.of(Sort.vectorize(vectorize)); + } + } + + if (sorts != null) { + return sorts.toArray(new Sort[0]); + } + return null; + } + + @SuppressWarnings("unchecked") + protected CollectionDefinition legacyGetCollectionDefinitionFromOp(ParsedOp op, long l) { CollectionDefinition optionsBldr = new CollectionDefinition(); Optional> dimFunc = op.getAsOptionalFunction("dimensions", Integer.class); if (dimFunc.isPresent()) { @@ -237,19 +629,24 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> allowFunc = op.getAsOptionalFunction("allowIndex", List.class); + @SuppressWarnings("unchecked") + Optional>> allowFunc = (Optional>>) (Optional) op.getAsOptionalFunction("allowIndex", List.class); if (allowFunc.isPresent()) { - LongFunction af = allowFunc.get(); - optionsBldr = optionsBldr.indexingAllow(Arrays.toString(af.apply(l).toArray(new String[0]))); + LongFunction> af = allowFunc.get(); + optionsBldr = optionsBldr.indexingAllow(af.apply(l).toArray(new String[0])); } - Optional> denyFunc = op.getAsOptionalFunction("denyIndex", List.class); + @SuppressWarnings("unchecked") + Optional>> denyFunc = (Optional>>) (Optional) op.getAsOptionalFunction("denyIndex", List.class); if (denyFunc.isPresent()) { - LongFunction df = denyFunc.get(); - optionsBldr = optionsBldr.indexingDeny(Arrays.toString(df.apply(l).toArray(new String[0]))); + LongFunction> df = denyFunc.get(); + optionsBldr = optionsBldr.indexingDeny(df.apply(l).toArray(new String[0])); } return optionsBldr; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionCountDocumentsOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionCountDocumentsOp.java index a7d7d416f..1977fb948 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionCountDocumentsOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionCountDocumentsOp.java @@ -22,12 +22,17 @@ import com.datastax.astra.client.collections.exceptions.TooManyDocumentsToCountE import com.datastax.astra.client.collections.definition.documents.Document; import com.datastax.astra.client.core.query.Filter; +import java.util.Optional; + public class DataApiCollectionCountDocumentsOp extends DataApiBaseOp { private final Collection collection; private final Filter filter; - private final int upperBound; + private final Optional upperBound; - public DataApiCollectionCountDocumentsOp(Database db, Collection collection, Filter filter, int upperBound) { + private final int MAX_UPPER_BOUND = 1000; + private final int SILENT_FAILED_COUNT_RESULT = -1; + + public DataApiCollectionCountDocumentsOp(Database db, Collection collection, Filter filter, Optional upperBound) { super(db); this.collection = collection; this.filter = filter; @@ -36,10 +41,21 @@ public class DataApiCollectionCountDocumentsOp extends DataApiBaseOp { @Override public Object apply(long value) { + // upper-bound being optional enables a pattern that ignores the "moreData" + // (getting rid of the 'upper bound' client-only feature). Still honored if passed (incl. throwing the error). + int upperBoundToUse = MAX_UPPER_BOUND; + boolean shouldThrowBoundExceeded = false; + if (upperBound.isPresent()) { + upperBoundToUse = upperBound.get(); + shouldThrowBoundExceeded = true; + } try { - return collection.countDocuments(filter, upperBound); + return collection.countDocuments(filter, upperBoundToUse); } catch (TooManyDocumentsToCountException e) { - throw new RuntimeException(e); + if (shouldThrowBoundExceeded) { + throw new RuntimeException(e); + } + return SILENT_FAILED_COUNT_RESULT; } } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindAndRerankOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindAndRerankOp.java new file mode 100644 index 000000000..2f22964b4 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindAndRerankOp.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.collections.Collection; +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.collections.commands.cursor.CollectionFindAndRerankCursor; +import com.datastax.astra.client.collections.commands.options.CollectionFindAndRerankOptions; + +public class DataApiCollectionFindAndRerankOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final CollectionFindAndRerankOptions options; + + public DataApiCollectionFindAndRerankOp(Database db, Collection collection, Filter filter, CollectionFindAndRerankOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.options = options; + } + + @Override + public Object apply(long value) { + CollectionFindAndRerankCursor cursor = collection.findAndRerank(filter, options); + return cursor.toList(); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindOneAndUpdateOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindOneAndUpdateOp.java index bae98b30c..d38c26d4a 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindOneAndUpdateOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindOneAndUpdateOp.java @@ -21,21 +21,24 @@ import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.collections.definition.documents.Document; import com.datastax.astra.client.core.query.Filter; import com.datastax.astra.client.collections.commands.Update; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneAndUpdateOptions; public class DataApiCollectionFindOneAndUpdateOp extends DataApiBaseOp { private final Collection collection; private final Filter filter; private final Update update; + private final CollectionFindOneAndUpdateOptions options; - public DataApiCollectionFindOneAndUpdateOp(Database db, Collection collection, Filter filter, Update update) { + public DataApiCollectionFindOneAndUpdateOp(Database db, Collection collection, Filter filter, Update update, CollectionFindOneAndUpdateOptions options) { super(db); this.collection = collection; this.filter = filter; this.update = update; + this.options = options; } @Override public Object apply(long value) { - return collection.findOneAndUpdate(filter, update); + return collection.findOneAndUpdate(filter, update, options); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyCountDocumentsOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyCountDocumentsOp.java new file mode 100644 index 000000000..1088fdd0e --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyCountDocumentsOp.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.collections.Collection; +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.collections.exceptions.TooManyDocumentsToCountException; +import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.core.query.Filter; + +public class DataApiCollectionLegacyCountDocumentsOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final int upperBound; + + public DataApiCollectionLegacyCountDocumentsOp(Database db, Collection collection, Filter filter, int upperBound) { + super(db); + this.collection = collection; + this.filter = filter; + this.upperBound = upperBound; + } + + @Override + public Object apply(long value) { + try { + return collection.countDocuments(filter, upperBound); + } catch (TooManyDocumentsToCountException e) { + throw new RuntimeException(e); + } + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionDeleteAllOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyDeleteAllOp.java similarity index 85% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionDeleteAllOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyDeleteAllOp.java index dbe5fa1c5..2f1538699 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionDeleteAllOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyDeleteAllOp.java @@ -19,10 +19,10 @@ package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.collections.Collection; import com.datastax.astra.client.databases.Database; -public class DataApiCollectionDeleteAllOp extends DataApiBaseOp { +public class DataApiCollectionLegacyDeleteAllOp extends DataApiBaseOp { private final Collection collection; - public DataApiCollectionDeleteAllOp(Database db, Collection collection) { + public DataApiCollectionLegacyDeleteAllOp(Database db, Collection collection) { super(db); this.collection = collection; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindByIdOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindByIdOp.java similarity index 86% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindByIdOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindByIdOp.java index 4161f539d..efb85e527 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindByIdOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindByIdOp.java @@ -19,11 +19,11 @@ package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.collections.Collection; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.collections.definition.documents.Document; -public class DataApiCollectionFindByIdOp extends DataApiBaseOp { +public class DataApiCollectionLegacyFindByIdOp extends DataApiBaseOp { private final Collection collection; private final Object id; - public DataApiCollectionFindByIdOp(Database db, Collection collection, Object id) { + public DataApiCollectionLegacyFindByIdOp(Database db, Collection collection, Object id) { super(db); this.collection = collection; this.id = id; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindDistinctOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindDistinctOp.java similarity index 82% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindDistinctOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindDistinctOp.java index 4c9aa78d8..a602d503d 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindDistinctOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindDistinctOp.java @@ -19,14 +19,15 @@ package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.collections.Collection; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.core.query.Filters; -public class DataApiCollectionFindDistinctOp extends DataApiBaseOp { +public class DataApiCollectionLegacyFindDistinctOp extends DataApiBaseOp { private final Collection collection; private final String fieldName; private final Filter filter; private final Class resultClass; - public DataApiCollectionFindDistinctOp(Database db, Collection collection, String fieldName, Filter filter, Class resultClass) { + public DataApiCollectionLegacyFindDistinctOp(Database db, Collection collection, String fieldName, Filter filter, Class resultClass) { super(db); this.collection = collection; this.fieldName = fieldName; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindOneAndUpdateOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindOneAndUpdateOp.java new file mode 100644 index 000000000..22c05ff40 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindOneAndUpdateOp.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.collections.Collection; +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.core.query.Filter; +import com.datastax.astra.client.collections.commands.Update; +import com.datastax.astra.client.collections.commands.options.CollectionFindOneAndUpdateOptions; + +public class DataApiCollectionLegacyFindOneAndUpdateOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final Update update; + private final CollectionFindOneAndUpdateOptions options; + + public DataApiCollectionLegacyFindOneAndUpdateOp(Database db, Collection collection, Filter filter, Update update, CollectionFindOneAndUpdateOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.update = update; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.findOneAndUpdate(filter, update, options); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindVectorFilterOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindVectorFilterOp.java similarity index 88% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindVectorFilterOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindVectorFilterOp.java index 38b4c629d..58d6527a9 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindVectorFilterOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindVectorFilterOp.java @@ -24,12 +24,12 @@ import com.datastax.astra.client.collections.definition.documents.Document; import com.datastax.astra.client.core.query.Filter; import com.datastax.astra.client.core.query.Sort; -public class DataApiCollectionFindVectorFilterOp extends DataApiBaseOp { +public class DataApiCollectionLegacyFindVectorFilterOp extends DataApiBaseOp { private final Collection collection; private final Filter filter; private final CollectionFindOptions options; - public DataApiCollectionFindVectorFilterOp(Database db, Collection collection, float[] vector, int limit, Filter filter) { + public DataApiCollectionLegacyFindVectorFilterOp(Database db, Collection collection, float[] vector, int limit, Filter filter) { super(db); this.collection = collection; this.filter = filter; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindVectorOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindVectorOp.java similarity index 88% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindVectorOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindVectorOp.java index 7d1536644..7dc43b3d6 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionFindVectorOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyFindVectorOp.java @@ -23,11 +23,11 @@ import com.datastax.astra.client.core.query.Sort; import com.datastax.astra.client.collections.commands.cursor.CollectionFindCursor; import com.datastax.astra.client.collections.commands.options.CollectionFindOptions; -public class DataApiCollectionFindVectorOp extends DataApiBaseOp { +public class DataApiCollectionLegacyFindVectorOp extends DataApiBaseOp { private final Collection collection; private final CollectionFindOptions options; - public DataApiCollectionFindVectorOp(Database db, Collection collection, float[] vector, int limit) { + public DataApiCollectionLegacyFindVectorOp(Database db, Collection collection, float[] vector, int limit) { super(db); this.collection = collection; this.options = new CollectionFindOptions().sort(Sort.vector(vector)).limit(limit); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionInsertOneVectorOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyInsertOneVectorOp.java similarity index 87% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionInsertOneVectorOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyInsertOneVectorOp.java index 1c0d4e635..38a763eac 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionInsertOneVectorOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyInsertOneVectorOp.java @@ -21,11 +21,11 @@ import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.collections.definition.documents.Document; import com.datastax.astra.client.collections.commands.results.CollectionInsertOneResult; -public class DataApiCollectionInsertOneVectorOp extends DataApiBaseOp { +public class DataApiCollectionLegacyInsertOneVectorOp extends DataApiBaseOp { private final Document doc; private final String collectionName; - public DataApiCollectionInsertOneVectorOp(Database db, String collectionName, Document doc, float[] vector) { + public DataApiCollectionLegacyInsertOneVectorOp(Database db, String collectionName, Document doc, float[] vector) { super(db); this.collectionName = collectionName; this.doc = doc.vector(vector); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionReplaceOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyReplaceOneOp.java similarity index 85% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionReplaceOneOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyReplaceOneOp.java index 26e254972..6041860e6 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionReplaceOneOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCollectionLegacyReplaceOneOp.java @@ -21,13 +21,13 @@ import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.core.query.Filter; import com.datastax.astra.client.collections.commands.options.CollectionReplaceOneOptions; -public class DataApiCollectionReplaceOneOp extends DataApiBaseOp { +public class DataApiCollectionLegacyReplaceOneOp extends DataApiBaseOp { private final Collection collection; private final Filter filter; private final Object replacement; private final CollectionReplaceOneOptions options; - public DataApiCollectionReplaceOneOp(Database db, Collection collection, Filter filter, Object replacement, CollectionReplaceOneOptions options) { + public DataApiCollectionLegacyReplaceOneOp(Database db, Collection collection, Filter filter, Object replacement, CollectionReplaceOneOptions options) { super(db); this.collection = collection; this.filter = filter; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDbCreateCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDbCreateCollectionOp.java new file mode 100644 index 000000000..1039562f8 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDbCreateCollectionOp.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.databases.Database; +import com.datastax.astra.client.collections.definition.documents.Document; +import com.datastax.astra.client.core.commands.Command; + +import java.util.Map; + +public class DataApiDbCreateCollectionOp extends DataApiBaseOp { + private final String collectionName; + private final Map definition; + + public DataApiDbCreateCollectionOp(Database db, String collectionName, Map definition) { + super(db); + this.collectionName = collectionName; + this.definition = definition; + } + + @Override + public Object apply(long value) { + Document ccPayload = new Document(); + ccPayload.append("name", collectionName); + ccPayload.append("options", definition); + Command ccCommand = new Command("createCollection", ccPayload); + return db.runCommand(ccCommand); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDbDeleteCollectionOp.java similarity index 90% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDbDeleteCollectionOp.java index feb5e406e..e3aabd52f 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDbDeleteCollectionOp.java @@ -18,9 +18,9 @@ package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.databases.Database; -public class DataApiDropCollectionOp extends DataApiBaseOp { +public class DataApiDbDeleteCollectionOp extends DataApiBaseOp { private final String collectionName; - public DataApiDropCollectionOp(Database db, String collectionName) { + public DataApiDbDeleteCollectionOp(Database db, String collectionName) { super(db); this.collectionName = collectionName; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyCreateCollectionOp.java similarity index 85% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyCreateCollectionOp.java index ef01b4618..6f1ee2a8e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyCreateCollectionOp.java @@ -19,11 +19,11 @@ package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.collections.definition.CollectionDefinition; -public class DataApiCreateCollectionOp extends DataApiBaseOp { +public class DataApiLegacyCreateCollectionOp extends DataApiBaseOp { private final String collectionName; private final CollectionDefinition definition; - public DataApiCreateCollectionOp(Database db, String collectionName, CollectionDefinition definition) { + public DataApiLegacyCreateCollectionOp(Database db, String collectionName, CollectionDefinition definition) { super(db); this.collectionName = collectionName; this.definition = definition; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyCreateCollectionWithClassOp.java similarity index 84% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyCreateCollectionWithClassOp.java index 4d21bbc57..b74e21785 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyCreateCollectionWithClassOp.java @@ -19,12 +19,12 @@ package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.databases.Database; import com.datastax.astra.client.collections.definition.CollectionDefinition; -public class DataApiCreateCollectionWithClassOp extends DataApiBaseOp { +public class DataApiLegacyCreateCollectionWithClassOp extends DataApiBaseOp { private final String collectionName; private final CollectionDefinition definition; private final Class clazz; - public DataApiCreateCollectionWithClassOp(Database db, String collectionName, CollectionDefinition definition, Class clazz) { + public DataApiLegacyCreateCollectionWithClassOp(Database db, String collectionName, CollectionDefinition definition, Class clazz) { super(db); this.collectionName = collectionName; this.definition = definition; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyDeleteCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyDeleteCollectionOp.java new file mode 100644 index 000000000..380a2403b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiLegacyDeleteCollectionOp.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.databases.Database; + +public class DataApiLegacyDeleteCollectionOp extends DataApiBaseOp { + private final String collectionName; + public DataApiLegacyDeleteCollectionOp(Database db, String collectionName) { + super(db); + this.collectionName = collectionName; + } + + @Override + public Object apply(long value) { + db.dropCollection(collectionName); + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index a9c8370ae..46a073a2e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -17,36 +17,69 @@ package io.nosqlbench.adapter.dataapi.ops; public enum DataApiOpType { - create_collection, - insert_many, - insert_one, - insert_one_vector, - find, - find_one, - find_one_and_delete, - find_one_and_update, - find_vector, - find_vector_filter, - update_one, - update_many, - delete_one, - delete_many, - delete_collection, - list_collections, - list_collection_names, - estimated_document_count, - find_by_id, - find_distinct, - count_documents, - replace_one, - find_one_and_replace, - delete_all, - create_collection_with_class, + + // TO-DO OPS + // admin ops: create_database, list_databases, drop_database, get_database_info, + // db-admin ops: create_namespace, list_namespaces, drop_namespace, + // in-database ops: + list_collections, + list_collection_names, + + // LEGACY OPS + // admin ops: + // db-admin ops: + // in-database ops: + create_collection, + create_collection_with_class, + delete_collection, + // in-collection ops: + insert_one, + insert_one_vector, + insert_many, + delete_one, + delete_many, + delete_all, + find_one_and_delete, + find, + find_vector, + find_vector_filter, + find_by_id, + find_distinct, + find_one, + update_one, + find_one_and_update, + replace_one, + find_one_and_replace, + update_many, + estimated_document_count, + count_documents, + + // NEW-STYLE OPS + // admin ops: + // db-admin ops: + // in-database ops: + db_create_collection, + db_delete_collection, + // in-collection ops: + collection_insert_one, + collection_insert_many, + collection_delete_many, + collection_delete_one, + collection_find_one_and_delete, + collection_find, + collection_find_one, + collection_find_and_rerank, + collection_update_one, + collection_find_one_and_update, + collection_find_one_and_replace, + collection_update_many, + collection_estimated_document_count, + collection_count_documents, } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/package-info.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/package-info.java new file mode 100644 index 000000000..72fd85a32 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/package-info.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// This package is the top-level entry point for the NoSQLBench Data API adapter (`dataapi`). +/// It wires together the standard adapter architecture: +/// {@link io.nosqlbench.adapter.dataapi.DataApiDriverAdapter} (registered via `@Service`), +/// {@link io.nosqlbench.adapter.dataapi.DataApiOpMapper} (dispatches to per-op dispensers), +/// and {@link io.nosqlbench.adapter.dataapi.DataApiSpace} (holds the driver client state). +/// +/// Op dispensers live in the `opdispensers` sub-package; their corresponding runnable ops +/// live in the `ops` sub-package. +package io.nosqlbench.adapter.dataapi; diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml deleted file mode 100644 index baeacb2b5..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml +++ /dev/null @@ -1,254 +0,0 @@ -min_version: "5.21.0" - -description: | - A basic workload that uses the DataStax Data API Client in Java, emulating what - applications would do in the native stack. - variations: - without vector: - with _id: - search by id= - search by id< - NOT USED: with uuid: - with vector: - with _id: - search by id= - search by id< - vector search with filter by id< - vector search with vector ~ - with uuid: - vector search with vector ~ - - cardinality=TEMPLATE(cardinality,1000) - collection=TEMPLATE(collection,keyvalue) - dimensions=TEMPLATE(dimensions,1536) - similarity=TEMPLATE(similarity,COSINE) - keycount=TEMPLATE(keycount,TEMPLATE(cardinality)) - valuecount=TEMPLATE(valuecount,TEMPLATE(cardinality)) - rampup-threads=TEMPLATE(rampup-threads,100) - - -scenarios: - - astra_dapi_kv_id: - schema: >- - run driver=dataapi - tags==block:schema_kv - threads==1 cycles==UNDEF - rampup: >- - run driver=dataapi - tags==block:rampup_kv_id - cycles===TEMPLATE(rampup-cycles,TEMPLATE(cardinality)) - threads=TEMPLATE(rampup-threads) - errors=count - find_kv_by_id: >- - run driver=dataapi - tags==block:find_kv_by_id - cycles===TEMPLATE(main-cycles,1000) - threads=auto errors=count - find_kv_lt_id: >- - run driver=dataapi - tags==block:find_kv_lt_id - cycles===TEMPLATE(main-cycles,1000) - threads=auto errors=count - - astra_dapi_v1536_id: - schema_vector: >- - run driver=dataapi - tags==block:schema_v1536 - threads==1 cycles==UNDEF - rampup_vector: >- - run driver=dataapi - tags==block:rampup_v1536_id - cycles===TEMPLATE(rampup-cycles,TEMPLATE(cardinality)) - threads=TEMPLATE(rampup-threads) - errors=count - find_kv_by_id: >- - run driver=dataapi - tags==block:find_kv_by_id - cycles===TEMPLATE(main-cycles,1000) - threads=auto errors=count - find_kv_lt_id: >- - run driver=dataapi - tags==block:find_kv_lt_id - cycles===TEMPLATE(main-cycles,1000) - threads=auto errors=count - find_by_vector: >- - run driver=dataapi - tags==block:find_by_vector - cycles===TEMPLATE(main-cycles,TEMPLATE(cardinality)) - threads=auto errors=count - find_by_vector_lt_id: >- - run driver=dataapi - tags==block:find_by_vector_lt_id - cycles===TEMPLATE(main-cycles,TEMPLATE(cardinality)) - threads=auto errors=count - - astra_dapi_v1536_uuid: - schema_vector: >- - run driver=dataapi - tags==block:schema_v1536 - threads==1 cycles==UNDEF - rampup_vector: >- - run driver=dataapi - tags==block:rampup_v1536_uuid - cycles===TEMPLATE(rampup-cycles,TEMPLATE(cardinality)) - threads=TEMPLATE(rampup-threads) - errors=count - find_by_vector: >- - run driver=dataapi - tags==block:find_by_vector - cycles===TEMPLATE(main-cycles,TEMPLATE(cardinality)) - threads=auto errors=count - -bindings: - - seq_key: Mod(TEMPLATE(keycount)); ToString() -> String - seq_value: Hash(); Mod(TEMPLATE(valuecount)); ToString() -> String - rw_key: TEMPLATE(keydist,Uniform(0,TEMPLATE(keycount))); ToString() -> String - rw_key_num: TEMPLATE(keydist,Uniform(0,TEMPLATE(keycount))); - rw_value: Hash(); TEMPLATE(valdist,Uniform(0,TEMPLATE(valuecount))); ToString() -> String - vector_value: HashedFloatVectors(TEMPLATE(dimensions,1536)); - request_id: ToHashedUUID(); ToString(); - -params: - cl: TEMPLATE(cl,LOCAL_QUORUM) - instrument: true - -blocks: - - reset_schema: - ops: - drop_index: - raw: |- - DROP INDEX IF EXISTS TEMPLATE(keyspace, baselines).TEMPLATE(table,keyvalue)_value_idx; - drop-table: - raw: |- - DROP TABLE IF EXISTS TEMPLATE(keyspace, baselines).TEMPLATE(table,keyvalue); - -# Schema - - schema_kv: - ops: - delete_collection_op: - delete_collection: "TEMPLATE(collection)" - create_collection_op: - create_collection: "TEMPLATE(collection)" - - schema_v1536: - ops: - delete_collection_op_v: - delete_collection: "TEMPLATE(collection)_v" - create_collection_op_v: - create_collection: "TEMPLATE(collection)_v" - dimensions: TEMPLATE(dimensions) - similarity: TEMPLATE(similarity) - -# Rampup - - rampup_kv_id: - ops: - insert_one_op: - insert_one: "TEMPLATE(collection)" - document: - _id: "{seq_key}" - value: "{seq_value}" - -# rampup_kv_uuid: -# ops: -# insert_one_op: -# insert_one: "TEMPLATE(collection)" -# document: -# value: "{seq_value}" - - - rampup_v1536_id: - ops: - insert_one_op_v: - insert_one_vector: "TEMPLATE(collection)_v" - document: - _id: "{seq_key}" - value: "{seq_value}" - vector: "{vector_value}" - - rampup_v1536_uuid: - ops: - insert_one_op_v: - insert_one_vector: "TEMPLATE(collection)_v" - document: - value: "{seq_value}" - vector: "{vector_value}" - - find_kv_by_id: - ops: - find_op: - find: "TEMPLATE(collection)" - filters: - - conjunction: "and" - operator: "eq" - field: "_id" - value: "{rw_key}" - - find_kv_lt_id: - params: - ratio: 5 - ops: - find_kv_id_lt: - find: "TEMPLATE(collection)" - filters: - - conjunction: "and" - operator: "lt" - field: "_id" - value: "{rw_key_num}" - - find_by_vector_lt_id: - params: - ratio: 5 - ops: - find_kv_id_lt: - find: "TEMPLATE(collection)" - filters: - - conjunction: "and" - operator: "lt" - field: "_id" - value: "{rw_key_num}" - vector: "{vector_value}" - - find_by_vector: - params: - ratio: 5 - ops: - find_op_filter_v: - find_vector_filter: "TEMPLATE(collection)" - vector: "{vector_value}" - -# main_ann_with_vector_limit_20: -# params: -# ratio: 5 -# ops: -# find_op: -# find: "TEMPLATE(collection)" -# sort: -# vector: "{vector_value}" -# options: -# limit: 20 -# schema_with_text_sai: -# ops: -# delete_collection_op: -# delete_collection: "TEMPLATE(collection)" -# create_collection_op: -# create_collection: "TEMPLATE(collection)" -# rampup_with_text_sai: -# ops: -# insert_one_op: -# insert_one: "TEMPLATE(collection)" -# document: -# _id: "{seq_key}" -# value: "{seq_value}" -# main_read_with_text_sai: -# params: -# ratio: 5 -# ops: -# find_op: -# find: "TEMPLATE(collection)" -# filter: -# value: "{rw_value}" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml deleted file mode 100644 index 66ebc5fdf..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml +++ /dev/null @@ -1,11 +0,0 @@ -scenarios: - default: - create_collection: run driver=dataapi tags==blocks:create_collection cycles=1 - -blocks: - create_collection: - ops: - op1: - create_collection: "collectionName" - dimensions: 10 - similarity: "COSINE" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/data_api_workload.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/data_api_workload.yaml new file mode 100644 index 000000000..52ab0aac5 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/data_api_workload.yaml @@ -0,0 +1,274 @@ +# Run with e.g.: +# +# nb data_api_workload.yaml \ +# astraApiEndpoint=$ASTRA_DB_API_ENDPOINT \ +# astraToken=$ASTRA_DB_APPLICATION_TOKEN \ +# namespace=$ASTRA_DB_KEYSPACE \ +# [main-cycles=] \ +# [rampup-insertmany-cycles=] +# +# where = data_api_scenario_base / data_api_scenario_vector + +description: | + A simple workload that uses the Data API Client to perform + writes, reads and vector reads similar to a realistic general-purpose application. + One scenario does not involve vectors, the other does. + The vector scenario demonstrates bulk insert (which can be done regardless of vector or not). + + data_api_scenario_base: + schema + rampup (single-document insertions) + find one by document _id + find one by another document field + find (many) with a inequality filter on a field + data_api_scenario_vector: + schema + rampup (single-document insertions) + rampup (fewer multi-document insert-many) + find one by document _id + find one by another document field + find (many) with a inequality filter on a field + vector ANN find + vector ANN find plus filter conditions on other fields + + NOTE: 'rampup-threads' cannot exceed 'cardinality' + + cardinality=TEMPLATE(cardinality,1000) + collection=TEMPLATE(collection,dataapi_collection) + dimension=TEMPLATE(dimension,1536) + similarity=TEMPLATE(similarity,cosine) + valuecount=TEMPLATE(valuecount,TEMPLATE(cardinality)) + rampup-threads=TEMPLATE(rampup-threads,100) + rampup-insertmany-cycles=TEMPLATE(rampup-insertmany-cycles,5) + +scenarios: + + data_api_scenario_base: + schema: >- + run driver=dataapi + tags==block:schema_base + threads==1 cycles==UNDEF + rampup: + run driver=dataapi + tags==block:rampup_base + cycles===TEMPLATE(rampup-cycles,TEMPLATE(cardinality)) + threads=TEMPLATE(rampup-threads) + errors=count + find_by_id: + run driver=dataapi + tags==block:find_by_id + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + find_by_value: + run driver=dataapi + tags==block:find_by_value + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + find_by_ineq: + run driver=dataapi + tags==block:find_by_ineq + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + + data_api_scenario_vector: + schema: >- + run driver=dataapi + tags==block:schema_vector + threads==1 cycles==UNDEF + rampup: + run driver=dataapi + tags==block:rampup_vector + cycles===TEMPLATE(rampup-cycles,TEMPLATE(cardinality)) + threads=TEMPLATE(rampup-threads) + errors=count + rampup_insertmany: + run driver=dataapi + tags==block:rampup_vector_insertmany + cycles===TEMPLATE(rampup-insertmany-cycles,TEMPLATE(cardinality)) + threads=TEMPLATE(rampup-threads) + errors=count + find_by_id: + run driver=dataapi + tags==block:find_by_id + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + find_by_value: + run driver=dataapi + tags==block:find_by_value + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + find_by_ineq: + run driver=dataapi + tags==block:find_by_ineq + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + vector_find: + run driver=dataapi + tags==block:vector_find + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + filtered_vector_find: + run driver=dataapi + tags==block:filtered_vector_find + cycles===TEMPLATE(main-cycles,1000) + threads=auto + errors=count + +bindings: + my_seq: Mod(TEMPLATE(cardinality)) + my_value: Hash(); ToString() -> String + my_id: Mod(TEMPLATE(cardinality)); ToString() -> String + my_vector: Add(100); HashedFloatVectors(TEMPLATE(dimension,1536)); + # vectors for batches of "insert_many": + ins_vector_00: Add(200); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_01: Add(201); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_02: Add(202); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_03: Add(203); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_04: Add(204); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_05: Add(205); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_06: Add(206); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_07: Add(207); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_08: Add(208); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_09: Add(209); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_10: Add(210); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_11: Add(211); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_12: Add(212); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_13: Add(213); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_14: Add(214); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_15: Add(215); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_16: Add(216); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_17: Add(217); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_18: Add(218); HashedFloatVectors(TEMPLATE(dimension,1536)); + ins_vector_19: Add(219); HashedFloatVectors(TEMPLATE(dimension,1536)); + +blocks: + + schema_base: + ops: + create_collection_base_op: + db_create_collection: "TEMPLATE(collection)" + definition: {} + + schema_vector: + ops: + create_collection_vector_op: + db_create_collection: "TEMPLATE(collection)" + definition: + vector: + dimension: TEMPLATE(dimension,1536) + metric: TEMPLATE(similarity,cosine) + + rampup_base: + ops: + insert_one_base_op: + collection_insert_one: "TEMPLATE(collection)" + document: + _id: "{my_id}" + some_value: "{my_value}" + seq: "{my_seq}" + + rampup_vector: + ops: + insert_one_vector_op: + collection_insert_one: "TEMPLATE(collection)" + document: + _id: "{my_id}" + some_value: "{my_value}" + seq: "{my_seq}" + $vector: "{my_vector}" + + rampup_vector_insertmany: + ops: + insert_many_vector_op: + collection_insert_many: "TEMPLATE(collection)" + documents: + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_00}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_01}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_02}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_03}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_04}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_05}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_06}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_07}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_08}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_09}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_10}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_11}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_12}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_13}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_14}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_15}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_16}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_17}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_18}"} + - {"some_value":"{my_value}","seq":"{my_seq}","$vector":"{ins_vector_19}"} + + find_by_id: + ops: + find_by_id_op: + collection_find_one: "TEMPLATE(collection)" + filter: + _id: "{my_id}" + projection: + include: + - "value" + - "seq" + + find_by_value: + ops: + find_by_value_op: + collection_find_one: "TEMPLATE(collection)" + filter: + some_value: "{my_value}" + projection: + include: + - "*" + + find_by_ineq: + ops: + find_by_ineq_op: + collection_find: "TEMPLATE(collection)" + filter: + seq: + $lt: "{my_seq}" + limit: 10 + sort: + seq: desc + projection: + include: + - "seq" + + vector_find: + ops: + vector_find_op: + collection_find: "TEMPLATE(collection)" + limit: 10 + vector: "{my_vector}" + include_similarity: true + projection: + include: + - "seq" + + filtered_vector_find: + ops: + vector_find_op: + collection_find: "TEMPLATE(collection)" + limit: 10 + vector: "{my_vector}" + filter: + seq: + $lt: "{my_seq}" + some_value: + $exists: true + projection: + include: + - "seq" + exclude: + - "_id" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml deleted file mode 100644 index 18f981631..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml +++ /dev/null @@ -1,18 +0,0 @@ -scenarios: - default: - delete_many: run driver=dataapi tags==blocks:delete_many cycles=1 - -blocks: - delete_many: - ops: - op1: - delete_many: "collectionName" - filters: - - conjunction: "and" - operator: "lt" - field: "field1" - value: 10 - - conjunction: "or" - operator: "eq" - field: "field2" - value: "value2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml deleted file mode 100644 index 2b9b7a799..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml +++ /dev/null @@ -1,13 +0,0 @@ -scenarios: - default: - delete_one: run driver=dataapi tags==blocks:delete_one cycles=1 - -blocks: - delete_one: - ops: - op1: - delete_one: "collectionName" - sort: - type: "asc" - field: "field1" - vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml deleted file mode 100644 index 434bc92a7..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml +++ /dev/null @@ -1,9 +0,0 @@ -scenarios: - default: - drop_collection: run driver=dataapi tags==blocks:drop_collection cycles=1 - -blocks: - drop_collection: - ops: - op1: - drop_collection: "collectionName" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/estimated_document_count.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/estimated_document_count.yaml deleted file mode 100644 index 21c7ea129..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/estimated_document_count.yaml +++ /dev/null @@ -1,9 +0,0 @@ -scenarios: - default: - estimated_document_count: run driver=dataapi tags==blocks:estimated_document_count cycles=1 - -blocks: - estimated_document_count: - ops: - op1: - estimated_document_count: "collectionName" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/example_workload.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/example_workload.yaml new file mode 100644 index 000000000..583000081 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/example_workload.yaml @@ -0,0 +1,47 @@ +scenarios: + default: + find_one: run driver=dataapi tags==blocks:find_one cycles=2 + +# Run with e.g.: +# nb example_workload.yaml \ +# astraApiEndpoint=$ASTRA_DB_API_ENDPOINT \ +# astraToken=$ASTRA_DB_APPLICATION_TOKEN \ +# namespace=$ASTRA_DB_KEYSPACE + +bindings: + query_vector: Add(37); HashedFloatVectors(TEMPLATE(dimensions,1024)); + +blocks: + find_one: + ops: + example_regular_find: + collection_find: "TEMPLATE(collection)" + filter: + $and: + - field: + $lt: 10 + - block: + $eq: "A" + sort: + idx: asc + limit: 5 + projection: + include: + - "idx" + - "block" + exclude: + - "_id" + + example_vector_find: + collection_find: "TEMPLATE(collection)" + vector: "{query_vector}" + # alternatively: + # vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0..." + # vector: [1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0...] + limit: 5 + projection: + include: + - "idx" + - "block" + exclude: + - "_id" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/example_workload_legacy.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/example_workload_legacy.yaml new file mode 100644 index 000000000..c0f8a3c31 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/example_workload_legacy.yaml @@ -0,0 +1,51 @@ +scenarios: + default: + find_one: run driver=dataapi tags==blocks:find_one cycles=2 + +# Run with e.g.: +# nb example_workload_legacy.yaml \ +# astraApiEndpoint=$ASTRA_DB_API_ENDPOINT \ +# astraToken=$ASTRA_DB_APPLICATION_TOKEN \ +# namespace=$ASTRA_DB_KEYSPACE + +bindings: + query_vector: Add(37); HashedFloatVectors(TEMPLATE(dimensions,1024)); + +blocks: + find_one: + ops: + legacy_example_regular_find: + find: "TEMPLATE(collection)" + filters: + - conjunction: "and" + operator: "lt" + field: "field" + value: 10 + - conjunction: "and" + operator: "eq" + field: "block" + value: "A" + sort: + type: "asc" + field: "idx" + limit: 5 + projection: + include: + - "idx" + - "block" + exclude: + - "_id" + + legacy_example_vector_find: + find: "TEMPLATE(collection)" + vector: "{query_vector}" + # alternatively: + # vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0..." + # vector: [1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0...] + limit: 5 + projection: + include: + - "idx" + - "block" + exclude: + - "_id" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml deleted file mode 100644 index 72b406dcd..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml +++ /dev/null @@ -1,26 +0,0 @@ -scenarios: - default: - find: run driver=dataapi tags==blocks:find cycles=10 - -blocks: - find: - ops: - op1: - find: "collectionName" - filters: - - conjunction: "and" - operator: "lt" - field: "field1" - value: 10 - - conjunction: "or" - operator: "eq" - field: "field2" - value: "value2" - sort: - type: "asc" - field: "field1" - vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" - projection: - include: - - "field1" - - "field2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml deleted file mode 100644 index e86bb16a3..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml +++ /dev/null @@ -1,26 +0,0 @@ -scenarios: - default: - find_one: run driver=dataapi tags==blocks:find_one cycles=10 - -blocks: - find_one: - ops: - op1: - find_one: "collectionName" - filters: - - conjunction: "and" - operator: "lt" - field: "field1" - value: 10 - - conjunction: "or" - operator: "eq" - field: "field2" - value: "value2" - sort: - type: "asc" - field: "field1" - vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" - projection: - include: - - "field1" - - "field2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml deleted file mode 100644 index e28b27c3c..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml +++ /dev/null @@ -1,25 +0,0 @@ -scenarios: - default: - find_one_and_delete: run driver=dataapi tags==blocks:find_one_and_delete cycles=1 - -blocks: - find_one_and_delete: - ops: - op1: - find_one_and_delete: "collectionName" - filters: - - conjunction: "and" - operator: "lt" - field: "field1" - value: 10 - - conjunction: "or" - operator: "eq" - field: "field2" - value: "value2" - sort: - type: "asc" - field: "field1" - projection: - include: - - "field1" - - "field2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_update.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_update.yaml deleted file mode 100644 index edff96902..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_update.yaml +++ /dev/null @@ -1,4 +0,0 @@ -min_version: "5.21.0" - -description: | - TBD, similar to find_one_and_delete.yaml diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml deleted file mode 100644 index 81a818d6f..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml +++ /dev/null @@ -1,17 +0,0 @@ -scenarios: - default: - insert_many: run driver=dataapi tags==blocks:insert_many cycles=1 - -blocks: - insert_many: - ops: - op1: - insert_many: "collectionName" - options: - chunkSize: "1000" - concurrency: "10" - ordered: "false" - documents: - - "{valid json here}" - - "{valid json here}" - - "{valid json here}" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml deleted file mode 100644 index 567f06cd0..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml +++ /dev/null @@ -1,10 +0,0 @@ -scenarios: - default: - insert_one: run driver=dataapi tags==blocks:insert_one cycles=1 - -blocks: - insert_one: - ops: - op1: - insert_one: "collectionName" - document: "{valid json here}" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml deleted file mode 100644 index a9bce8200..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml +++ /dev/null @@ -1,11 +0,0 @@ -scenarios: - default: - insert_one: run driver=dataapi tags==blocks:insert_one cycles=1 - -blocks: - insert_one: - ops: - op1: - insert_one: "collectionName" - document: "{valid json here}" - vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/legacy_ops_reference.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/legacy_ops_reference.yaml new file mode 100644 index 000000000..187df015d --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/legacy_ops_reference.yaml @@ -0,0 +1,407 @@ +# NOTE: this file only lists the (legacy) operations and their syntax in a self-documenting comment-rich form. +# It is not a full NB workload file. Refer to the full examples for the overall format. + +blocks: + my_block: + ops: + + # LEGACY "deleteCollection", fully documented: + delcoll_legacy: + delete_collection: "TEMPLATE(collection)" + + # LEGACY "createCollection", fully documented: + crcoll_legacy: + create_collection: "TEMPLATE(collection)" + ### optional vector dimensionality (if present, it's a vector collection. Note plural, 'dimensions'): + # dimensions: 1024 + #### optional similarity function for vector (case-insensitive "COSINE"/"EUCLIDEAN"/"DOT_PRODUCT"): + # similarity: DOT_PRODUCT + #### optional (required for vectorize) service provider: + # serviceProvider: voyageAI + #### optional (required for vectorize) service model (note a typo, 'mode'): + # serviceMode: voyage-2 + #### optional KMS secret name for vectorize API key: + # serviceSharedSecretKey: SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI + #### optional parameters map for vectorize service/model: + # serviceParameters: + # autoTruncate: true + #### optional default ID type (case-sensitive "objectId"/"uuidv6"/"uuidv7"/"uuid"): + # collectionType: objectId + #### optional, allow-list for indexing (error if denyIndex present as well): + # allowIndex: + # - aField1 + # - aField2 + #### optional, deny-list for indexing (error if allowIndex present as well): + # denyIndex: + # - aField1 + # - aField2 + + # LEGACY "insertMany", fully documented: + im_legacy: + insert_many: "TEMPLATE(collection)" + # required documents to insert (a list of strings, each a serialized JSON): + documents: + - "{\"a\": 1}" + - "{\"a\": 2}" + - "{\"a\": 3}" + #### optional, options (each must be a string if provided): + # options: + # # how to split the input list into separate requests (default 50): + # chunkSize: "20" + # # concurrency level for running chunk insertions (must be 1 if ordered=true): + # concurrency: "5" + # # ordered insertion? true = insert sequentially and fail fast; false = no order guarantees, proceed even if errors: + # ordered: "false" + # # timeout for each individual chunk insertion (milliseconds): + # timeout: "5000" + + # LEGACY "insertOne", fully documented (note: identical to new-style op except in the op name): + i1_legacy: + insert_one: "TEMPLATE(collection)" + #### required document to insert, free-form (_id can be omitted; note alternate forms for optional $vector): + document: + new: true + # $vector: [1,2,3,4,5,6,7,8,9,0] + # $vector: + # $binary: "P4AAAD+AAAA/gAAAP4AAAD+AAAA/gAAAP4AAAD+AAAA/gAAAP4AAAA==" + + # LEGACY "insertOne" with explicit vector, fully documented: + i1vec_legacy: + insert_one_vector: "TEMPLATE(collection)" + #### required document to insert, free-form (_id can be omitted, $vector is ignored): + document: + new: true + # required vector value to come with the document (three alternate forms): + vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + # vector: "1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0" + # vector: + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + # - 1.0 + + # LEGACY "updateMany", fully documented: + um_legacy: + update_many: "TEMPLATE(collection)" + #### required update, free-form: + updates: + $set: + updated: "yes!" + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "old" + # value: true + #### optional upsert: + # upsert: true + + # LEGACY replace_one, fully documented: + repl1_legacy: + replace_one: "TEMPLATE(collection)" + #### required replacement document in its entirety, free-form (_id can be omitted): + document: + new: true + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "old" + # value: true + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (either include or exclude, list of fields): + # projection: + # exclude: + # - "_id" + #### optional upsert: + # upsert: true + #### optional 'returnDocument' (values: 'after'/'before') + # returnDocument: after + + # LEGACY "findOneAndReplace". Fully documented: + f1repl_legacy: + find_one_and_replace: "TEMPLATE(collection)" + #### required replacement document in its entirety, free-form (_id can be omitted): + document: + new: true + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "idx" + # value: 20 + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (either include or exclude, list of fields): + # projection: + # exclude: + # - "idx" + #### optional upsert: + # upsert: true + #### optional 'returnDocument' (values: 'after'/'before') + # returnDocument: after + + # LEGACY "findOneAndDelete". Fully documented: + f1del_legacy: + find_one_and_delete: "TEMPLATE(collection)" + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "idx" + # value: 20 + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (either include or exclude, list of fields): + # projection: + # exclude: + # - "idx" + + # LEGACY "updateOne". Fully documented: + u1_legacy: + update_one: "TEMPLATE(collection)" + #### required update (free-form): + update: + $set: + updated: true + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "idx" + # value: 29 + #### optional upsert: + # upsert: true + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + + # LEGACY countDocuments. Fully documented: + cd_legacy: + count_documents: "TEMPLATE(collection)" + #### required upper bound (a client-side notion. Error thrown if result exceeds it): + upperbound: 100 + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "lt" + # field: "idx" + # value: 100 + + # LEGACY estimatedDocumentCount (only the op name differs from 'new'). Fully documented: + edc_legacy: + estimated_document_count: "TEMPLATE(collection)" + + # LEGACY delete-all, i.e. "deleteMany" with no filters (truncate). Fully documented: + da_legacy: + delete_all: "TEMPLATE(collection_to_truncate)" + + # LEGACY "deleteMany". Fully documented: + dm_legacy: + delete_many: "TEMPLATE(collection)" + #### required filter (either all 'and' or all 'or', cannot be empty): + filters: + - conjunction: "and" + operator: "gt" + field: "idx" + value: 52 + + # LEGACY "deleteOne". Fully documented: + d1_legacy: + delete_one: "TEMPLATE(collection)" + #### required filter (either all 'and' or all 'or', cannot be empty): + filters: + - conjunction: "and" + operator: "lt" + field: "idx" + value: 100 + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + + # LEGACY "find (no vectors)". Fully documented: + find_legacy: + find: "TEMPLATE(collection)" + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "lt" + # field: "idx" + # value: 80 + #### optional limit: + # limit: 40 + #### optional skip: + # skip: 1 + #### optional includeSimilarity (default to true in this op): + # includeSimilarity: true + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (either include or exclude, list of fields): + # projection: + # include: + # - "idx" + #### optional pageState to start the find from: + # pageState: "LQAAAAEBAAAAJDc1NDgyNzk5LTQwN2ItNGEyOC04ODI3LTk5NDA3YmFhMjg4YQDwf///zQA=" + + # LEGACY "find with vector". Fully documented: + find_vector_legacy: + find_vector: "TEMPLATE(collection)" + #### required sort vector (float list/string repr with commas): + vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional, limit (defaults to 100): + # limit: 3 + + # LEGACY "find with vector and a filter". Fully documented: + find_vector_filter_legacy: + find_vector_filter: "TEMPLATE(collection)" + #### required sort vector (float list/string repr with commas): + vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "lt" + # field: "idx" + # value: 80 + #### optional, limit (defaults to 100): + # limit: 3 + + # LEGACY "findOne (by ID)". Fully documented: + find_by_id_legacy: + find_by_id: "TEMPLATE(collection)" + #### required id of the target document: + id: "zzz" + + # LEGACY "findOne". Fully documented: + f1_legacy: + find_one: "TEMPLATE(collection)" + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "field1" + # value: 74 + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (either include or exclude, list of fields): + # projection: + # include: + # - "field1" + # - "field2" + #### optional includeSimilarity (default to true in this op): + # include_similarity: false + + # LEGACY "findOneAndUpdate". Fully documented: + f1upd_legacy: + find_one_and_update: "TEMPLATE(collection)" + #### required update (can only 1 update op, legacy op limitation): + updates: + update: + operation: inc + field: "incd field" + value: 500 + #### optional sort (can only 1 sort, not if 'vector/ize'): + # sort: + # type: "asc" + # field: "idx" + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "field1" + # value: 74 + #### optional upsert: + # upsert: true + + # LEGACY find-distinct (client-side operation). Fully documented: + find_distinct_op_legacy: + find_distinct: "TEMPLATE(collection)" + #### required field to read: + fieldName: "the_field" + #### optional filter (either all 'and' or all 'or'): + # filters: + # - conjunction: "and" + # operator: "eq" + # field: "field1" + # value: 74 + #### optional class for the result (default: java.lang.String) + # resultClass: "java.lang.Double" + + # LEGACY create collection 'with class' (the 'with class' is a client-side operation only). Fully documented: + create_collection_with_class_legacy: + create_collection_with_class: "TEMPLATE(collection_to_create)" + #### required class name for the collection instance returned (default is shown): + createClass: "com.datastax.astra.client.collections.definition.documents.Document" + ### optional vector dimensionality (if present, it's a vector collection. Note plural, 'dimensions'): + # dimensions: 1024 + #### optional similarity function for vector (case-insensitive "COSINE"/"EUCLIDEAN"/"DOT_PRODUCT"): + # similarity: DOT_PRODUCT + #### optional (required for vectorize) service provider: + # serviceProvider: voyageAI + #### optional (required for vectorize) service model (note a typo, 'mode'): + # serviceMode: voyage-2 + #### optional KMS secret name for vectorize API key: + # serviceSharedSecretKey: SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI + #### optional parameters map for vectorize service/model: + # serviceParameters: + # autoTruncate: true + #### optional default ID type (case-sensitive "objectId"/"uuidv6"/"uuidv7"/"uuid"): + # collectionType: objectId + #### optional, allow-list for indexing (error if denyIndex present as well): + # allowIndex: + # - aField1 + # - aField2 + #### optional, deny-list for indexing (error if allowIndex present as well): + # denyIndex: + # - aField1 + # - aField2 diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/list_collection_names.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/list_collection_names.yaml deleted file mode 100644 index 4234ccc58..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/list_collection_names.yaml +++ /dev/null @@ -1,9 +0,0 @@ -scenarios: - default: - list_collection_names: run driver=dataapi tags==blocks:list_collection_names cycles=1 - -blocks: - list_collection_names: - ops: - op1: - list_collection_names: TBD diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/list_collections.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/list_collections.yaml deleted file mode 100644 index 87d2b9ac8..000000000 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/list_collections.yaml +++ /dev/null @@ -1,9 +0,0 @@ -scenarios: - default: - list_collections: run driver=dataapi tags==blocks:list_collections cycles=1 - -blocks: - list_collections: - ops: - op1: - list_collections: TBD diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/ops_reference.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/ops_reference.yaml new file mode 100644 index 000000000..06984295e --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/ops_reference.yaml @@ -0,0 +1,341 @@ +# NOTE: this file only lists the operations and their syntax in a self-documenting comment-rich form. +# It is not a full NB workload file. Refer to the full examples for the overall format. + +blocks: + my_block: + ops: + + # NEW "deleteCollection", fully documented: + delcoll: + db_delete_collection: "TEMPLATE(collection)" + + # NEW "createCollection", fully documented: + crcoll: + db_create_collection: "TEMPLATE(collection)" + #### required collection definition (free-form, must be valid Data API syntax) + definition: + vector: + dimension: 1024 + metric: "dot_product" + service: + provider: "voyageAI" + modelName: "voyage-2" + authentication: + providerKey: "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI" + parameters: + autoTruncate: true + indexing: + allow: + - aField1 + - aField2 + defaultId: + type: objectId + + # NEW "insertMany", fully documented: + im: + collection_insert_many: "TEMPLATE(collection)" + # required documents to insert, list of free-form objects: + documents: + - {"a": 1} + - a: 2 + b: my_string + c: + c1: subitem1 + c2: 1999 + - a: 3 + $vector: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1] + #### optional ordered flag, default=false (true = insert sequentially and fail fast; false = no order guarantees, proceed even if errors): + # ordered: false + #### optional for splitting the input list into separate requests (default 50): + #### NOTE: in a typical NoSQLBench workload, this should not count and not be used, meaning the amount of documents + #### submitted should be <= chunk size. This is because NB supposedly works in unit of requests to the server, + #### not mediated by manipulations from the Data API client. + # chunk_size: 20 + + # NEW "insertOne", fully documented: + i1: + collection_insert_one: "TEMPLATE(collection)" + #### required document to insert, free-form (_id can be omitted; note alternate forms for optional $vector): + document: + newv2: true + # $vector: [1,2,3,4,5,6,7,8,9,0] + + # NEW "updateMany", fully documented: + um: + collection_update_many: "TEMPLATE(collection)" + #### (Note: may fail to update more than first page until fix to https://github.com/datastax/astra-db-java/issues/106 is released) + #### required update (free-form): + update: + $set: + updated: "yes!" + #### optional filter (free-form): + # filter: + # $and: + # - marker_field: present + # - not_a_field: + # $exists: false + #### optional upsert: + # upsert: true + + # NEW "findOneAndReplace". Fully documented: + f1repl: + collection_find_one_and_replace: "TEMPLATE(collection)" + #### required replacement document in its entirety, free-form (_id can be omitted): + replacement: + new: true + #### optional filter (free-form): + # filter: + # $and: + # - _id: + # $ne: abcd0123 + # - not_a_field: + # $exists: false + #### optional upsert: + # upsert: true + #### optional 'return_document' (values: 'after'/'before') + # return_document: after + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (lists of fields; include and exclude can coexist only for special fields): + # projection: + # exclude: + # - "idx" + + # NEW "findOneAndDelete". Fully documented: + f1del: + collection_find_one_and_delete: "TEMPLATE(collection)" + #### optional filter (free-form): + # filter: + # $and: + # - idx: + # $gte: 21 + # - not_a_field: + # $exists: false + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (lists of fields; include and exclude can coexist only for special fields): + # projection: + # exclude: + # - "idx" + + # NEW "updateOne". Fully documented: + u1: + collection_update_one: "TEMPLATE(collection)" + #### required update (free-form): + update: + $set: + updated: true + #### optional filter (free-form): + # filter: + # $and: + # - idx: 29 + # - not_a_field: + # $exists: false + #### optional upsert: + # upsert: true + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + + # NEW "countDocuments". Fully documented: + cd: + collection_count_documents: "TEMPLATE(collection)" + #### optional filter (free-form): + # filter: + # fake_field: + # $exists: false + #### optional upper bound (a client-side notion. *If provided*, error thrown when exceeded) + # upper_bound: 100 + + # NEW "estimatedDocumentCount". Fully documented: + edc: + collection_estimated_document_count: "TEMPLATE(collection)" + + # NEW "deleteMany". Fully documented: + dm: + collection_delete_many: "TEMPLATE(collection)" + #### required filter (free-form. truncate with 'filter: {}') + filter: + idx: + $gt: 40 + + # NEW "deleteOne". Fully documented: + d1: + collection_delete_one: "TEMPLATE(collection)" + #### required filter (free-form): + filter: + $and: + - idx: + $gte: 45 + - the_field: + $exists: false + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + + # NEW "find". Fully documented: + f: + collection_find: "TEMPLATE(collection)" + #### optional filter (free-form): + # filter: + # $or: + # - idx: + # $lt: 99 + # - idx: + # $exists: false + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional limit: + # limit: 36 + #### optional skip (only with non-vector sorting): + # skip: 45 + #### optional projection (lists of fields; include and exclude can coexist only for special fields): + # projection: + # include: + # - "idx" + # - "bloppo" + #### optional includeSimilarity (default true for nb): + # include_similarity: false + #### optional includeSortVector (default false): + # include_sort_vector: true + + # NEW "findOne". Fully documented: + f1: + collection_find_one: "TEMPLATE(collection)" + #### optional filter (free-form): + # filter: + # $and: + # - field1: "value1" + # - field2: + # $eq: 123 + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional projection (lists of fields; include and exclude can coexist only for special fields): + # projection: + # include: + # - "field1" + # - "field2" + #### optional includeSimilarity: + # include_similarity: false + + # NEW "findOneAndUpdate". Fully documented: + f1upd: + collection_find_one_and_update: "TEMPLATE(collection)" + #### required update (free-form): + update: + $set: + myfield2: "new_val" + $inc: + myfield: 200 + myfield1: 20 + #### optional sort (order matters; can't use if 'vector/ize'): + # sort: + # block: asc + # idx: desc + #### optional vector (not if 'sort/vectorize'; float list/string repr with commas): + # vector: [1,2,3,4,5,6,7,8,9,0] + #### optional vectorize (not if 'sort/vector'): + # vectorize: "query string" + #### optional filter (free-form): + # filter: + # $and: + # - field1: "value1" + # - field2: + # $eq: 123 + #### optional upsert: + # upsert: true + + # NEW "findAndRerank", fully documented: + farr: + collection_find_and_rerank: "TEMPLATE(collection)" + #### required sort clause. Must have '$hybrid' key, with one of the various forms accepted (also depending on collection settings): + sort: + $hybrid: bla + ## alternative (explicit per-leg) sort form: + # sort: + # $hybrid: + # $vectorize: bli + # $lexical: blo + ## alternative (explicit per-leg) sort form: + # sort: + # $hybrid: + # $vector: [0.1,0.2,0.3] + # $lexical: "bla" + ## alternative (no-lexical) sort form: + # sort: + # $hybrid: + # $vectorize: bli + ## alternative (no-lexical) sort form: + # sort: + # $hybrid: + # $vector: [0.1,0.2,0.3] + #### optional filter (free-form): + # filter: + # $vectorize: + # $exists: true + #### optional projection (lists of fields; include and exclude can coexist only for special fields): + # projection: + # include: + # - $vectorize + # exclude: + # - _id + #### optional limit: + # limit: 1 + #### optional hybridLimits (can have either form): + # hybrid_limits: 10 + ## alternative (explicit per-leg) hybridLimits form: + # hybrid_limits: + # $vector: 11 + # $lexical: 9 + #### optional rerankOn: + # rerank_on: "$lexical" + #### optional rerankQuery: + # rerank_query: "ble" + #### optional includeScores (default false): + # include_scores: true + #### optional includeSortVector (default false): + # include_sort_vector: true + #### optional rerank override for this one call: + #### (Note: may not have effects until fix to https://github.com/datastax/astra-db-java/issues/109 released) + # rerank: + # provider: theProvider" + # modelName: "theModelName" + # authentication: + # providerKey: "bla" + # parameters: + # param1: value1 + # param2: 999 diff --git a/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md b/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md index 7c7f9f02e..e9d117051 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md +++ b/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md @@ -1,6 +1,82 @@ --- source: nb-adapters/adapter-dataapi/src/main/resources/dataapi.md --- -# Data API -## DataStax Data API Adapter +# Data API adapter + +This is a NB adapter for the [Data API](https://github.com/stargate/data-api), an HTTP API that can run on top of Cassandra and DataStax Astra DB. + +The adapter is built on top of the [Java client](https://github.com/datastax/astra-db-java) for the Data API and exposes the same primitives as the Data API, in a 1:1 relation between ops and Data API commands wherever possible and exposing all of the command options when it makes sense to do so. + +## Command-line parameters + +_Note: this release of the adapter is designed to connect to **DataStax Astra DB's Data API** specifically. All examples refer to this case. Subsequent releases will expand the scope to non-Astra-DB instances of Data API._ + +Connection to the Data API requires an **API Endpoint** and an **Application Token**, plus specification of a **keyspace** to operate in. These are required parameters when launching a NB workload targeting the Data API: + +``` +astraToken="..." +astraApiEndpoint="https://..." +namespace="the_keyspace" +``` + +These values are used by the adapter to establish a `DataAPIClient`, from which all subsequent operations are spawned. + +## Legacy vs. "new-style" operation + +In September 2026, a major rework of the ops has been initiated, essentially marking all operations existing so far as "legacy" and introducing a whole new set of ops, coming with more extensible naming conventions (e.g. the legacy `find` op has become `collection_find`). The overhaul has reached its first milestone (collection CRUD and all collection operations). + +The legacy operations received bug fixes (plus obvious additive extensions to their behaviour when it made sense to). So, no pre-existing workload will break. However, the user is encouraged to build new workload exclusively using the new-style operations. + +For a detailed catalog of the legacy and the new-style operations, with their syntax, consult the "Examples" section below. + +### Current status of the overhaul + +**Done:** + +- all collection DML ops; +- create/delete collection DDL; +- All known bugs in legacy operations are fixed. + +**Coming next:** + +- introduction of Table support; +- Administrative functions (create keyspace and so on), currently left as they were before the overhaul; +- Ability to target non-Astra and non-prod environments in full; +- Ability to supply special Data API-related secrets (such as embedding provider API keys for $vectorize operations). + +## Usage notes (new-style ops) + +Most commands support a free-form syntax for some of their parameters, such as the collection definition for `db_create_collection` or the insertee document in `collection_insert_one`. This is to give the most flexibility and future-proofing the op in case new syntax is added by the Data API. + +In other cases, when it was deemed more convenient, the syntax is more constrained. All specifications are given in the supporting example op YAMLs under `activities/`. + +### Vectors + +Many commands allow to supply **vectors**: these can be expressed in the YAML: + +- literally, as lists of (floating-point) numbers: `[0.1,-3.2,1.9]`; +- literally, as a string containing comma-separated floating points: `"0.1,-3.2,1.9"`; +- via an appropriate binding, e.g. `"{the_vector_binding}"`. + +In any case, whenever the API Client is equipped to do so, vectors are transmitted in the bandwidth-optimized binary-encoded form regardless of how they are given. One _should not_ manually construct the binary-encoding syntax (e.g. as part of an insertee document for a `collection_insert_one` op). + +## Client vs. 'raw Data API payloads' + +An important virtue of the Java Data API client is that it handles headers, authentication, encoding/decoding of payloads and responses - with minimal overhead. + +On the other hand, the client builds higher-level convenience features on top of the raw HTTP requests underlying the API: so, for example, what for the client is a single 200-item `insertMany(...)` invocation translates to a certain number of "chunked" Data API requests, each one inserting some of the items and executed with a degree of (client-managed) concurrency that depends on various settings. _This is a kind of sophistication that a typical usage of NB would not want_. + +For this reason, the adapter strives to avoid patterns that would trigger such "too complex" behaviours: for instance, the client concurrency for multiple insertions is always hardcoded to one, leaving management of concurrent requests purely on the NB side. + +It is, however, an ultimate responsibility of the Data API-savvy NB user to ensure full compliance with this tenet, for example never packing more than a pageful of items into a `collection_insert_many` operation's `documents` property. + +## Examples + +Consult the self-documenting YAML examples here: + +- a simple [minimal running workload](activities/example_workload.yaml); + - (a simple [legacy minimal running workload](activities/example_workload_legacy.yaml)); +- a realistic, [ready-to-run workload](activities/data_api_workload.yaml) demonstrating several ops +- [new-style ops full reference](activities/ops_reference.yaml); + - ([legacy ops full reference](activities/legacy_ops_reference.yaml)).