From 92c9cf474c6fc5279d30a30b1ef200ea98a68426 Mon Sep 17 00:00:00 2001 From: sahankj2000 Date: Mon, 13 May 2024 16:41:55 +0530 Subject: [PATCH 01/19] making changes --- .../cli/NBCLIScenarioPreprocessorTest.java | 7 +++++++ .../api/scenarios/NBCLIScenarioPreprocessor.java | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java index bb9b02156..77caa82a9 100644 --- a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java +++ b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java @@ -201,4 +201,11 @@ public class NBCLIScenarioPreprocessorTest { .isThrownBy(() -> new NBCLIOptions(new String[]{"scenario_test", "duplicate_param"}, NBCLIOptions.Mode.ParseAllOptions)) .withMessageContaining("Duplicate occurrence of parameter \"threads\""); } + + @Test + public void testThatSuggestionsAreShownForDirectStepNameUsage() { + assertThatExceptionOfType(BasicError.class) + .isThrownBy(() -> new NBCLIOptions(new String[]{"scenario_test", "schema"}, NBCLIOptions.Mode.ParseAllOptions)) + .withMessageContainingAll("default.schema", "schema_only.schema"); + } } diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java index 25a34a28c..41bdf0c35 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java @@ -133,8 +133,20 @@ public class NBCLIScenarioPreprocessor { if (nameparts.length==1) { Map namedScenario = scenarios.getNamedScenario(scenarioName); if (namedScenario==null) { - throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName - + "', but you can pick from one of: " + String.join(", ", scenarios.getScenarioNames())); + List matchingSteps = new LinkedList<>(); + for (String scenario : scenarios.getScenarioNames()) { + Map selectedScenario = scenarios.getNamedScenario(scenario); + if (selectedScenario.containsKey(scenarioName)) { + matchingSteps.add(scenario + "." + scenarioName); + } + } + if (matchingSteps.isEmpty()) { + throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName + + "', but you can pick from one of: " + String.join(", ", scenarios.getScenarioNames())); + } else { + throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName + + "', you might be looking for one of these scenario steps: " + String.join(", ", matchingSteps)); + } } namedSteps.putAll(namedScenario); } else { From 37cd2f7bb4a9ceda3c232e2328b52940ab1139bd Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Mon, 10 Jun 2024 19:20:13 -0400 Subject: [PATCH 02/19] add timeout to InsertManyOptions configuration --- .../dataapi/opdispensers/DataApiInsertManyOpDispenser.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java index 10788cfea..d51789523 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java @@ -66,6 +66,8 @@ public class DataApiInsertManyOpDispenser extends DataApiOpDispenser { options = options.concurrency(Integer.parseInt(entry.getValue())); case "ordered" -> options = options.ordered(Boolean.parseBoolean(entry.getValue())); + case "timeout" -> + options = options.timeout(Integer.parseInt(entry.getValue())); } } } From 93dfaec7437ab5a3b82a530a148952f067556d64 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 11 Jun 2024 12:56:32 -0400 Subject: [PATCH 03/19] new DataApiFindById op type --- .../adapter/dataapi/DataApiOpMapper.java | 1 + .../DataApiFindByIdOpDispenser.java | 62 +++++++++++++++++++ .../dataapi/ops/DataApiFindByIdOp.java | 38 ++++++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 1 + 4 files changed, 102 insertions(+) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindByIdOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindByIdOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index d075f0838..cdad5e57d 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -64,6 +64,7 @@ public class DataApiOpMapper implements OpMapper { new DataApiListCollectionNamesOpDispenser(adapter, op, typeAndTarget.targetFunction); case estimated_document_count -> new DataApiEstimatedDocumentCountOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_by_id -> new DataApiFindByIdOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindByIdOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindByIdOpDispenser.java new file mode 100644 index 000000000..01469723f --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindByIdOpDispenser.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneOptions; +import com.datastax.astra.client.model.Projection; +import com.datastax.astra.client.model.Sort; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindByIdOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiFindByIdOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindByIdOpDispenser.class); + private final LongFunction opFunction; + public DataApiFindByIdOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Object id = getIdFromOp(op, l); + return new DataApiFindByIdOp( + db, + db.getCollection(targetFunction.apply(l)), + id + ); + }; + } + + private Object getIdFromOp(ParsedOp op, long l) { + return op.getAsRequiredFunction("id", Object.class).apply(l); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindByIdOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindByIdOp.java new file mode 100644 index 000000000..603854512 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindByIdOp.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; + +public class DataApiFindByIdOp extends DataApiBaseOp { + private final Collection collection; + private final Object id; + + public DataApiFindByIdOp(Database db, Collection collection, Object id) { + super(db); + this.collection = collection; + this.id = id; + } + + @Override + public Object apply(long value) { + return collection.findById(id); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index fb1ac4a84..99577c3c2 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -35,4 +35,5 @@ public enum DataApiOpType { list_collections, list_collection_names, estimated_document_count, + find_by_id, } From f37796b8a0d433e574bda7df04f27a537c634ce9 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 11 Jun 2024 13:18:02 -0400 Subject: [PATCH 04/19] updated FindOptions to include all possible options --- .../opdispensers/DataApiFindOpDispenser.java | 18 ++++++++++++++++- .../DataApiFindVectorFilterOpDispenser.java | 20 ------------------- .../DataApiFindVectorOpDispenser.java | 17 ---------------- 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index d615cc656..a965f05a7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -28,6 +28,7 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOpDispenser extends DataApiOpDispenser { @@ -67,7 +68,22 @@ public class DataApiFindOpDispenser extends DataApiOpDispenser { if (projection != null) { options = options.projection(projection); } - options.setIncludeSimilarity(true); + Optional> limitFunction = op.getAsOptionalFunction("limit", Integer.class); + if (limitFunction.isPresent()) { + options = options.limit(limitFunction.get().apply(l)); + } + Optional> skipFunction = op.getAsOptionalFunction("skip", Integer.class); + if (skipFunction.isPresent()) { + options = options.skip(skipFunction.get().apply(l)); + } + Optional> includeSimilarityFunction = op.getAsOptionalFunction("includeSimilarity", Boolean.class); + if (includeSimilarityFunction.isPresent()) { + options.setIncludeSimilarity(includeSimilarityFunction.get().apply(l)); + } + Optional> pageStateFunction = op.getAsOptionalFunction("pageState", String.class); + if (pageStateFunction.isPresent()) { + options.setPageState(pageStateFunction.get().apply(l)); + } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorFilterOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorFilterOpDispenser.java index 3f30c5130..b010b0a14 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorFilterOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorFilterOpDispenser.java @@ -18,9 +18,6 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; -import com.datastax.astra.client.model.FindOptions; -import com.datastax.astra.client.model.Projection; -import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiFindVectorFilterOp; @@ -58,23 +55,6 @@ public class DataApiFindVectorFilterOpDispenser extends DataApiOpDispenser { return op.getConfigOr("limit", 100, l); } - private FindOptions getFindOptions(ParsedOp op, long l) { - FindOptions options = new FindOptions(); - Sort sort = getSortFromOp(op, l); - float[] vector = getVectorValues(op, l); - if (sort != null) { - options = vector != null ? options.sort(vector, sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(vector); - } - Projection[] projection = getProjectionFromOp(op, l); - if (projection != null) { - options = options.projection(projection); - } - options.setIncludeSimilarity(true); - return options; - } - @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java index 5aa65f912..5ddaca3a0 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java @@ -55,23 +55,6 @@ public class DataApiFindVectorOpDispenser extends DataApiOpDispenser { return op.getConfigOr("limit", 100, l); } - private FindOptions getFindOptions(ParsedOp op, long l) { - FindOptions options = new FindOptions(); - Sort sort = getSortFromOp(op, l); - float[] vector = getVectorValues(op, l); - if (sort != null) { - options = vector != null ? options.sort(vector, sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(vector); - } - Projection[] projection = getProjectionFromOp(op, l); - if (projection != null) { - options = options.projection(projection); - } - options.setIncludeSimilarity(true); - return options; - } - @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); From 1aa8d9ec37e799ca99e250d319ce9497da499e67 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Wed, 12 Jun 2024 08:57:52 -0400 Subject: [PATCH 05/19] update one op impl --- .../DataApiUpdateOneOpDispenser.java | 50 ++++++++++++++++++- .../dataapi/ops/DataApiUpdateOneOp.java | 43 ++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateOneOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java index 114c7945e..168521f50 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java @@ -16,19 +16,67 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.*; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiUpdateOneOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiUpdateOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiUpdateOneOpDispenser.class); + private final LongFunction opFunction; + public DataApiUpdateOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + UpdateOneOptions options = getUpdateOneOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("update", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + + return new DataApiUpdateOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + new Update(docFunc.apply(l)), + options + ); + }; + } + + private UpdateOneOptions getUpdateOneOptions(ParsedOp op, long l) { + UpdateOneOptions options = new UpdateOneOptions(); + Sort sort = getSortFromOp(op, l); + float[] vector = getVectorFromOp(op, l); + + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + if (sort != null) { + options = (vector != null) ? options.vector(vector, sort) : options.sort(sort); + } + return options; + } + + private float[] getVectorFromOp(ParsedOp op, long l) { + return getVectorValues(op.get("vector", l)); } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateOneOp.java new file mode 100644 index 000000000..8ee6f7d63 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateOneOp.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Update; +import com.datastax.astra.client.model.UpdateOneOptions; + +public class DataApiUpdateOneOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final Update update; + private final UpdateOneOptions options; + + public DataApiUpdateOneOp(Database db, Collection collection, Filter filter, Update update, UpdateOneOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.update = update; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.updateOne(filter, update, options); + } +} From c5f74e688cf09093a1de0af296d3ee86f8454248 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Wed, 12 Jun 2024 09:38:05 -0400 Subject: [PATCH 06/19] update many op impl --- .../DataApiUpdateManyOpDispenser.java | 40 ++++++++++++++++- .../DataApiUpdateOneOpDispenser.java | 5 +-- .../dataapi/ops/DataApiUpdateManyOp.java | 43 +++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateManyOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java index 82269b156..253e22349 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java @@ -16,19 +16,57 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.*; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiUpdateManyOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiUpdateManyOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiUpdateManyOpDispenser.class); + private final LongFunction opFunction; + public DataApiUpdateManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + UpdateManyOptions options = getUpdateManyOptions(op, l); + LongFunction> docMapFunc = op.getAsRequiredFunction("updates", Map.class); + + return new DataApiUpdateManyOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + new Update(docMapFunc.apply(l)), + options + ); + }; + } + + private UpdateManyOptions getUpdateManyOptions(ParsedOp op, long l) { + UpdateManyOptions options = new UpdateManyOptions(); + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + + return options; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java index 168521f50..88c2ab6e7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java @@ -43,14 +43,13 @@ public class DataApiUpdateOneOpDispenser extends DataApiOpDispenser { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); UpdateOneOptions options = getUpdateOneOptions(op, l); - LongFunction docMapFunc = op.getAsRequiredFunction("update", Map.class); - LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + LongFunction> docMapFunc = op.getAsRequiredFunction("update", Map.class); return new DataApiUpdateOneOp( db, db.getCollection(targetFunction.apply(l)), filter, - new Update(docFunc.apply(l)), + new Update(docMapFunc.apply(l)), options ); }; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateManyOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateManyOp.java new file mode 100644 index 000000000..4eaa75427 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiUpdateManyOp.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Update; +import com.datastax.astra.client.model.UpdateManyOptions; + +public class DataApiUpdateManyOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final Update update; + private final UpdateManyOptions options; + + public DataApiUpdateManyOp(Database db, Collection collection, Filter filter, Update update, UpdateManyOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.update = update; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.updateMany(filter, update, options); + } +} From 92f96f6bb40b36521867f397c5cacbed2ff6488b Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Wed, 12 Jun 2024 10:48:08 -0400 Subject: [PATCH 07/19] Find Distinct op --- .../adapter/dataapi/DataApiOpMapper.java | 1 + .../DataApiFindDistinctOpDispenser.java | 67 +++++++++++++++++++ .../DataApiUpdateManyOpDispenser.java | 2 +- .../DataApiUpdateOneOpDispenser.java | 2 +- .../dataapi/ops/DataApiFindDistinctOp.java | 42 ++++++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 1 + 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindDistinctOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindDistinctOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index cdad5e57d..2703f8aad 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -65,6 +65,7 @@ public class DataApiOpMapper implements OpMapper { case estimated_document_count -> new DataApiEstimatedDocumentCountOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_by_id -> new DataApiFindByIdOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_distinct -> new DataApiFindDistinctOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindDistinctOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindDistinctOpDispenser.java new file mode 100644 index 000000000..37eba4d89 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindDistinctOpDispenser.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindDistinctOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiFindDistinctOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindDistinctOpDispenser.class); + private final LongFunction opFunction; + public DataApiFindDistinctOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + Class targetClass = getTargetClass(op, l); + return new DataApiFindDistinctOp( + db, + db.getCollection(targetFunction.apply(l)), + op.getAsRequiredFunction("fieldName", String.class).apply(l), + filter, + targetClass + ); + }; + } + + private Class getTargetClass(ParsedOp op, long l) { + String className = op.getAsFunctionOr("resultClass", "java.lang.String").apply(l); + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java index 253e22349..98b525499 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java @@ -43,7 +43,7 @@ public class DataApiUpdateManyOpDispenser extends DataApiOpDispenser { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); UpdateManyOptions options = getUpdateManyOptions(op, l); - LongFunction> docMapFunc = op.getAsRequiredFunction("updates", Map.class); + LongFunction docMapFunc = op.getAsRequiredFunction("updates", Map.class); return new DataApiUpdateManyOp( db, diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java index 88c2ab6e7..9eca42fef 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java @@ -43,7 +43,7 @@ public class DataApiUpdateOneOpDispenser extends DataApiOpDispenser { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); UpdateOneOptions options = getUpdateOneOptions(op, l); - LongFunction> docMapFunc = op.getAsRequiredFunction("update", Map.class); + LongFunction docMapFunc = op.getAsRequiredFunction("update", Map.class); return new DataApiUpdateOneOp( db, diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindDistinctOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindDistinctOp.java new file mode 100644 index 000000000..898e2da81 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindDistinctOp.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.CollectionOptions; +import com.datastax.astra.client.model.Filter; + +public class DataApiFindDistinctOp extends DataApiBaseOp { + private final Collection collection; + private final String fieldName; + private final Filter filter; + private final Class resultClass; + + public DataApiFindDistinctOp(Database db, Collection collection, String fieldName, Filter filter, Class resultClass) { + super(db); + this.collection = collection; + this.fieldName = fieldName; + this.filter = filter; + this.resultClass = resultClass; + } + + @Override + public Object apply(long value) { + return collection.distinct(fieldName, filter, resultClass); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index 99577c3c2..caf50cac7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -36,4 +36,5 @@ public enum DataApiOpType { list_collection_names, estimated_document_count, find_by_id, + find_distinct, } From 70cd4a0dd4894a0b0640bbc4824d7c07fc5d6914 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Wed, 12 Jun 2024 11:51:30 -0400 Subject: [PATCH 08/19] impl of replace ops --- .../adapter/dataapi/DataApiOpMapper.java | 3 + .../DataApiCountDocumentsOpDispenser.java | 58 ++++++++++++ .../DataApiFindOneAndReplaceOpDispenser.java | 92 +++++++++++++++++++ .../DataApiReplaceOneOpDispenser.java | 74 +++++++++++++++ .../dataapi/ops/DataApiCountDocumentsOp.java | 46 ++++++++++ .../ops/DataApiFindOneAndReplaceOp.java | 42 +++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 3 + .../dataapi/ops/DataApiReplaceOneOp.java | 42 +++++++++ 8 files changed, 360 insertions(+) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCountDocumentsOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndReplaceOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiReplaceOneOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCountDocumentsOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndReplaceOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiReplaceOneOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 2703f8aad..c4539be8d 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -66,6 +66,9 @@ public class DataApiOpMapper implements OpMapper { new DataApiEstimatedDocumentCountOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_by_id -> new DataApiFindByIdOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_distinct -> new DataApiFindDistinctOpDispenser(adapter, op, typeAndTarget.targetFunction); + case count_documents -> new DataApiCountDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); + case replace_one -> new DataApiReplaceOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one_and_replace -> new DataApiFindOneAndReplaceOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCountDocumentsOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCountDocumentsOpDispenser.java new file mode 100644 index 000000000..09459c2c3 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCountDocumentsOpDispenser.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCountDocumentsOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCountDocumentsOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCountDocumentsOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCountDocumentsOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + int upperBound = op.getAsRequiredFunction("upperbound", Integer.class).apply(l); + + return new DataApiCountDocumentsOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + upperBound + ); + }; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndReplaceOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndReplaceOpDispenser.java new file mode 100644 index 000000000..7dd835a4e --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndReplaceOpDispenser.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.*; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneAndReplaceOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiFindOneAndReplaceOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindOneAndReplaceOpDispenser.class); + private final LongFunction opFunction; + + public DataApiFindOneAndReplaceOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + FindOneAndReplaceOptions options = getFindOneAndReplaceOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + + return new DataApiFindOneAndReplaceOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + docFunc.apply(l), + options + ); + }; + } + + private FindOneAndReplaceOptions getFindOneAndReplaceOptions(ParsedOp op, long l) { + FindOneAndReplaceOptions options = new FindOneAndReplaceOptions(); + Sort sort = getSortFromOp(op, l); + if (op.isDefined("vector")) { + float[] vector = getVectorValues(op, l); + if (sort != null) { + options = vector != null ? options.sort(vector, sort) : options.sort(sort); + } else if (vector != null) { + options = options.sort(vector); + } + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + if (op.isDefined("returnDocument")) { + options = switch ((String) op.get("returnDocument", l)) { + case "after" -> options.returnDocumentAfter(); + case "before" -> options.returnDocumentBefore(); + default -> throw new RuntimeException("Invalid returnDocument value: " + op.get("returnDocument", l)); + }; + } + return options; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiReplaceOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiReplaceOneOpDispenser.java new file mode 100644 index 000000000..144969125 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiReplaceOneOpDispenser.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.*; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiReplaceOneOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiUpdateOneOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Optional; +import java.util.function.LongFunction; + +public class DataApiReplaceOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiReplaceOneOpDispenser.class); + private final LongFunction opFunction; + + public DataApiReplaceOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + ReplaceOneOptions options = getReplaceOneOptions(op, l); + LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + + return new DataApiReplaceOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + docFunc.apply(l), + options + ); + }; + } + + private ReplaceOneOptions getReplaceOneOptions(ParsedOp op, long l) { + ReplaceOneOptions options = new ReplaceOneOptions(); + + Optional> upsertFunction = op.getAsOptionalFunction("upsert", Boolean.class); + if (upsertFunction.isPresent()) { + options = options.upsert(upsertFunction.get().apply(l)); + } + return options; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCountDocumentsOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCountDocumentsOp.java new file mode 100644 index 000000000..dfb27bba8 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCountDocumentsOp.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.exception.TooManyDocumentsToCountException; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneAndDeleteOptions; + +public class DataApiCountDocumentsOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final int upperBound; + + public DataApiCountDocumentsOp(Database db, Collection collection, Filter filter, int upperBound) { + super(db); + this.collection = collection; + this.filter = filter; + this.upperBound = upperBound; + } + + @Override + public Object apply(long value) { + try { + return collection.countDocuments(filter, upperBound); + } catch (TooManyDocumentsToCountException e) { + throw new RuntimeException(e); + } + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndReplaceOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndReplaceOp.java new file mode 100644 index 000000000..903fc3b94 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndReplaceOp.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneAndReplaceOptions; + +public class DataApiFindOneAndReplaceOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final Object replacement; + private final FindOneAndReplaceOptions options; + + public DataApiFindOneAndReplaceOp(Database db, Collection collection, Filter filter, Object replacement, FindOneAndReplaceOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.replacement = replacement; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.findOneAndReplace(filter, replacement, options); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index caf50cac7..fdaf746ae 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -37,4 +37,7 @@ public enum DataApiOpType { estimated_document_count, find_by_id, find_distinct, + count_documents, + replace_one, + find_one_and_replace, } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiReplaceOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiReplaceOneOp.java new file mode 100644 index 000000000..b35f4338e --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiReplaceOneOp.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.ReplaceOneOptions; + +public class DataApiReplaceOneOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final Object replacement; + private final ReplaceOneOptions options; + + public DataApiReplaceOneOp(Database db, Collection collection, Filter filter, Object replacement, ReplaceOneOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.replacement = replacement; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.replaceOne(filter, replacement, options); + } +} From 4618f4d405e06c860d21d7ded14f7e1d7ee442ad Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Wed, 12 Jun 2024 16:57:27 -0400 Subject: [PATCH 09/19] impl of delete all and create collection with class --- .../adapter/dataapi/DataApiOpMapper.java | 2 + .../DataApiCreateCollectionOpDispenser.java | 18 +----- ...iCreateCollectionWithClassOpDispenser.java | 61 +++++++++++++++++++ .../DataApiDeleteAllOpDispenser.java | 53 ++++++++++++++++ .../opdispensers/DataApiOpDispenser.java | 15 +++++ .../DataApiCreateCollectionWithClassOp.java | 38 ++++++++++++ .../dataapi/ops/DataApiDeleteAllOp.java | 35 +++++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 2 + 8 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteAllOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteAllOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index c4539be8d..58c5b114f 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -69,6 +69,8 @@ public class DataApiOpMapper implements OpMapper { case count_documents -> new DataApiCountDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); case replace_one -> new DataApiReplaceOneOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_one_and_replace -> new DataApiFindOneAndReplaceOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_all -> new DataApiDeleteAllOpDispenser(adapter, op, typeAndTarget.targetFunction); + case create_collection_with_class -> new DataApiCreateCollectionWithClassOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java index 14e30e5f1..f9ec6ea1b 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java @@ -39,26 +39,12 @@ public class DataApiCreateCollectionOpDispenser extends DataApiOpDispenser { private LongFunction createOpFunction(ParsedOp op) { return (l) -> { - CollectionOptions.CollectionOptionsBuilder optionsBldr = CollectionOptions.builder(); - Optional> dimFunc = op.getAsOptionalFunction("dimensions", Integer.class); - if (dimFunc.isPresent()) { - LongFunction af = dimFunc.get(); - optionsBldr.vectorDimension(af.apply(l)); - } -// COSINE("cosine"), -// EUCLIDEAN("euclidean"), -// DOT_PRODUCT("dot_product"); - Optional> simFunc = op.getAsOptionalFunction("similarity", String.class); - if (simFunc.isPresent()) { - LongFunction sf = simFunc.get(); - optionsBldr.vectorSimilarity(SimilarityMetric.fromValue(sf.apply(l))); - } - DataApiCreateCollectionOp dataApiCreateCollectionOp = new DataApiCreateCollectionOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), - optionsBldr.build()); + this.getCollectionOptionsFromOp(op, l) + ); return dataApiCreateCollectionOp; }; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java new file mode 100644 index 000000000..3b72cbb03 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionWithClassOpDispenser.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCreateCollectionWithClassOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCreateCollectionWithClassOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCreateCollectionWithClassOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCreateCollectionWithClassOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiCreateCollectionWithClassOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + this.getCollectionOptionsFromOp(op, l), + getCreateClass(op, l) + ); + } + + private Class getCreateClass(ParsedOp op, long l) { + String className = op.getAsFunctionOr("createClass", "com.datastax.astra.client.model.Document").apply(l); + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } + + +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteAllOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteAllOpDispenser.java new file mode 100644 index 000000000..a38a15dc4 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteAllOpDispenser.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteAllOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiDeleteAllOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDeleteAllOpDispenser.class); + private final LongFunction opFunction; + + public DataApiDeleteAllOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + + return new DataApiDeleteAllOp( + db, + db.getCollection(targetFunction.apply(l)) + ); + }; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index d9de2c62e..e516d1aa4 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -188,4 +188,19 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> dimFunc = op.getAsOptionalFunction("dimensions", Integer.class); + if (dimFunc.isPresent()) { + LongFunction af = dimFunc.get(); + optionsBldr.vectorDimension(af.apply(l)); + } + Optional> simFunc = op.getAsOptionalFunction("similarity", String.class); + if (simFunc.isPresent()) { + LongFunction sf = simFunc.get(); + optionsBldr.vectorSimilarity(SimilarityMetric.fromValue(sf.apply(l))); + } + return optionsBldr.build(); + } + } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java new file mode 100644 index 000000000..2bd5ef746 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionWithClassOp.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.CollectionOptions; + +public class DataApiCreateCollectionWithClassOp extends DataApiBaseOp { + private final String collectionName; + private final CollectionOptions options; + private final Class clazz; + + public DataApiCreateCollectionWithClassOp(Database db, String collectionName, CollectionOptions options, Class clazz) { + super(db); + this.collectionName = collectionName; + this.options = options; + this.clazz = clazz; + } + + @Override + public Object apply(long value) { + return db.createCollection(collectionName, options, clazz); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteAllOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteAllOp.java new file mode 100644 index 000000000..57e6c48c1 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteAllOp.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; + +public class DataApiDeleteAllOp extends DataApiBaseOp { + private final Collection collection; + + public DataApiDeleteAllOp(Database db, Collection collection) { + super(db); + this.collection = collection; + } + + @Override + public Object apply(long value) { + return collection.deleteAll(); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index fdaf746ae..dfba0096e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -40,4 +40,6 @@ public enum DataApiOpType { count_documents, replace_one, find_one_and_replace, + delete_all, + create_collection_with_class, } From f772af9d0e3bb70020c3c840957c902be8f47392 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Thu, 13 Jun 2024 11:28:54 -0400 Subject: [PATCH 10/19] adding full set of create collection options --- .../opdispensers/DataApiOpDispenser.java | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index e516d1aa4..466cfdc36 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -23,10 +23,7 @@ import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.function.LongFunction; public abstract class DataApiOpDispenser extends BaseOpDispenser { @@ -193,13 +190,39 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> dimFunc = op.getAsOptionalFunction("dimensions", Integer.class); if (dimFunc.isPresent()) { LongFunction af = dimFunc.get(); - optionsBldr.vectorDimension(af.apply(l)); + optionsBldr = optionsBldr.vectorDimension(af.apply(l)); } Optional> simFunc = op.getAsOptionalFunction("similarity", String.class); if (simFunc.isPresent()) { LongFunction sf = simFunc.get(); - optionsBldr.vectorSimilarity(SimilarityMetric.fromValue(sf.apply(l))); + optionsBldr = optionsBldr.vectorSimilarity(SimilarityMetric.fromValue(sf.apply(l))); } + Optional> typeFunc = op.getAsOptionalFunction("collectionType", String.class); + if (typeFunc.isPresent()) { + LongFunction tf = typeFunc.get(); + optionsBldr = optionsBldr.defaultIdType(CollectionIdTypes.fromValue(tf.apply(l))); + } + Optional> providerFunc = op.getAsOptionalFunction("serviceProvider", String.class); + Optional> modeFunc = op.getAsOptionalFunction("serviceMode", String.class); + Optional> paramFunc = op.getAsOptionalFunction("serviceParameters", Map.class); + if (providerFunc.isPresent() && modeFunc.isPresent()) { + LongFunction pf = providerFunc.get(); + LongFunction mf = modeFunc.get(); + optionsBldr = paramFunc.isPresent() ? + optionsBldr.vectorize(pf.apply(l), mf.apply(l), paramFunc.get().apply(l)) : + optionsBldr.vectorize(pf.apply(l), mf.apply(l)); + } + Optional> allowFunc = op.getAsOptionalFunction("allowIndex", List.class); + if (allowFunc.isPresent()) { + LongFunction af = allowFunc.get(); + optionsBldr = optionsBldr.indexingAllow(Arrays.toString(af.apply(l).toArray(new String[0]))); + } + Optional> denyFunc = op.getAsOptionalFunction("denyIndex", List.class); + if (denyFunc.isPresent()) { + LongFunction df = denyFunc.get(); + optionsBldr = optionsBldr.indexingDeny(Arrays.toString(df.apply(l).toArray(new String[0]))); + } + return optionsBldr.build(); } From f1ef3a24c13dc979d352d485c05fbadfe906022c Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 14 Jun 2024 15:17:11 -0400 Subject: [PATCH 11/19] adding administrative function ops --- .../adapter/dataapi/DataApiOpMapper.java | 3 + .../adapter/dataapi/DataApiSpace.java | 45 ++++++++++--- .../DataApiCreateDatabaseOpDispenser.java | 67 +++++++++++++++++++ .../DataApiDropDatabaseOpDispenser.java | 49 ++++++++++++++ .../DataApiListDatabasesOpDispenser.java | 47 +++++++++++++ .../adapter/dataapi/ops/DataApiAdminOp.java | 33 +++++++++ .../dataapi/ops/DataApiCreateDatabaseOp.java | 42 ++++++++++++ .../dataapi/ops/DataApiDropDatabaseOp.java | 41 ++++++++++++ .../dataapi/ops/DataApiListDatabasesOp.java | 33 +++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 3 + 10 files changed, 355 insertions(+), 8 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateDatabaseOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropDatabaseOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListDatabasesOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiAdminOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateDatabaseOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropDatabaseOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListDatabasesOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 58c5b114f..69b5f59d7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -71,6 +71,9 @@ public class DataApiOpMapper implements OpMapper { case find_one_and_replace -> new DataApiFindOneAndReplaceOpDispenser(adapter, op, typeAndTarget.targetFunction); case delete_all -> new DataApiDeleteAllOpDispenser(adapter, op, typeAndTarget.targetFunction); case create_collection_with_class -> new DataApiCreateCollectionWithClassOpDispenser(adapter, op, typeAndTarget.targetFunction); + case create_database -> new DataApiCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); + case list_databases -> new DataApiListDatabasesOpDispenser(adapter, op, typeAndTarget.targetFunction); + case drop_database -> new DataApiDropDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index e1cd61d30..8602f3789 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.dataapi; import com.datastax.astra.client.DataAPIClient; import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; import io.nosqlbench.nb.api.config.standard.ConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfiguration; @@ -42,10 +43,14 @@ public class DataApiSpace { private Database database; private String namespace; + private String superUserToken; + private AstraDBAdmin admin; + public DataApiSpace(String name, NBConfiguration cfg) { this.config = cfg; this.name = name; setToken(); + setSuperToken(); setApiEndpoint(); setNamespace(); createClient(); @@ -59,6 +64,10 @@ public class DataApiSpace { return database; } + public AstraDBAdmin getAdmin() { + return admin; + } + private void createClient() { this.dataAPIClient = new DataAPIClient(astraToken); if (namespace != null) { @@ -66,6 +75,11 @@ public class DataApiSpace { } else { this.database = dataAPIClient.getDatabase(astraApiEndpoint); } + if (superUserToken != null) { + this.admin = dataAPIClient.getAdmin(superUserToken); + } else { + this.admin = dataAPIClient.getAdmin(); + } } private void setApiEndpoint() { @@ -100,18 +114,33 @@ public class DataApiSpace { String tokenFileContents = null; Optional tokenFilePath = config.getOptional("astraTokenFile"); if (tokenFilePath.isPresent()) { - Path path = Paths.get(tokenFilePath.get()); - try { - tokenFileContents = Files.readAllLines(path).getFirst(); - } catch (IOException e) { - String error = "Error while reading token from file:" + path; - logger.error(error, e); - throw new RuntimeException(e); - } + tokenFileContents = getTokenFileContents(tokenFilePath.get()); } this.astraToken = (tokenFileContents != null) ? tokenFileContents : config.get("astraToken"); } + private String getTokenFileContents(String filePath) { + Path path = Paths.get(filePath); + try { + return Files.readAllLines(path).getFirst(); + } catch (IOException e) { + String error = "Error while reading token from file:" + path; + logger.error(error, e); + throw new RuntimeException(e); + } + } + + private void setSuperToken() { + String superTokenFileContents = null; + Optional superTokenFilePath = config.getOptional("superTokenFile"); + if (superTokenFilePath.isPresent()) { + superTokenFileContents = getTokenFileContents(superTokenFilePath.get()); + } + Optional maybeSuperToken = config.getOptional("superToken"); + // It's fine if this is null + this.superUserToken = maybeSuperToken.orElse(superTokenFileContents); + } + public static NBConfigModel getConfigModel() { return ConfigModel.of(DataApiSpace.class) .add( diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateDatabaseOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateDatabaseOpDispenser.java new file mode 100644 index 000000000..adbb7cf0b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateDatabaseOpDispenser.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.dtsx.astra.sdk.db.domain.CloudProviderType; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCreateDatabaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCreateDatabaseOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCreateDatabaseOpDispenser.class); + private final LongFunction opFunction; + public DataApiCreateDatabaseOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiCreateDatabaseOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getAdmin(), + targetFunction.apply(l), + getCloudProvider(op, l), + getRegion(op, l), + getWaitUntilActive(op, l) + ); + } + + /* + * These default to the same values used in the API if only the name is provided. + */ + private CloudProviderType getCloudProvider(ParsedOp op, long l) { + return CloudProviderType.valueOf(op.getAsFunctionOr("cloudProvider", "GCP").apply(l)); + } + + private String getRegion(ParsedOp op, long l) { + return op.getAsFunctionOr("region", "us-east1").apply(l); + } + + private boolean getWaitUntilActive(ParsedOp op, long l) { + return op.getAsFunctionOr("waitUntilActive", "true").apply(l).equals("true"); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropDatabaseOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropDatabaseOpDispenser.java new file mode 100644 index 000000000..2e4ee0596 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropDatabaseOpDispenser.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDropDatabaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiListDatabasesOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiDropDatabaseOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDropDatabaseOpDispenser.class); + private final LongFunction opFunction; + public DataApiDropDatabaseOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiDropDatabaseOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getAdmin(), + targetFunction.apply(l) + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListDatabasesOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListDatabasesOpDispenser.java new file mode 100644 index 000000000..42f6fdb71 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListDatabasesOpDispenser.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiListDatabasesOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiListDatabasesOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiListDatabasesOpDispenser.class); + private final LongFunction opFunction; + public DataApiListDatabasesOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiListDatabasesOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getAdmin() + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiAdminOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiAdminOp.java new file mode 100644 index 000000000..d54ebffd4 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiAdminOp.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public abstract class DataApiAdminOp extends DataApiBaseOp { + protected static final Logger logger = LogManager.getLogger(DataApiAdminOp.class); + protected final AstraDBAdmin admin; + + public DataApiAdminOp(Database db, AstraDBAdmin admin) { + super(db); + this.admin = admin; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateDatabaseOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateDatabaseOp.java new file mode 100644 index 000000000..9353aa9bb --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateDatabaseOp.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; +import com.dtsx.astra.sdk.db.domain.CloudProviderType; + +public class DataApiCreateDatabaseOp extends DataApiAdminOp { + private final String name; + private final CloudProviderType cloud; + private final String cloudRegion; + private final boolean waitForDb; + + public DataApiCreateDatabaseOp(Database db, AstraDBAdmin admin, String name, CloudProviderType cloud, String cloudRegion, boolean waitForDb) { + super(db, admin); + this.name = name; + this.cloud = cloud; + this.cloudRegion = cloudRegion; + this.waitForDb = waitForDb; + } + + @Override + public Object apply(long value) { + return admin.createDatabase(name, cloud, cloudRegion, waitForDb); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropDatabaseOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropDatabaseOp.java new file mode 100644 index 000000000..724e7e378 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropDatabaseOp.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; + +import java.util.UUID; +import java.util.regex.Pattern; + +public class DataApiDropDatabaseOp extends DataApiAdminOp { + private static final Pattern UUID_PATTERN = Pattern.compile( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"); + private final String nameOrUUID; + private final boolean isUUID; + + public DataApiDropDatabaseOp(Database db, AstraDBAdmin admin, String nameOrUUID) { + super(db, admin); + this.nameOrUUID = nameOrUUID; + this.isUUID = UUID_PATTERN.matcher(nameOrUUID).matches(); + } + + @Override + public Object apply(long value) { + return isUUID ? admin.dropDatabase(UUID.fromString(nameOrUUID)) : admin.dropDatabase(nameOrUUID); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListDatabasesOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListDatabasesOp.java new file mode 100644 index 000000000..39b31db2a --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListDatabasesOp.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; +import com.dtsx.astra.sdk.db.domain.CloudProviderType; + +public class DataApiListDatabasesOp extends DataApiAdminOp { + + public DataApiListDatabasesOp(Database db, AstraDBAdmin admin) { + super(db, admin); + } + + @Override + public Object apply(long value) { + return admin.listDatabases(); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index dfba0096e..634116e8e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -42,4 +42,7 @@ public enum DataApiOpType { find_one_and_replace, delete_all, create_collection_with_class, + create_database, + list_databases, + drop_database, } From 3f2a304b83a6b0f9a4d8e447c5ddff892764efa3 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 14 Jun 2024 15:52:43 -0400 Subject: [PATCH 12/19] finished administrative function ops --- .../adapter/dataapi/DataApiOpMapper.java | 4 ++ .../adapter/dataapi/DataApiSpace.java | 8 +++ .../DataApiCreateNamespaceOpDispenser.java | 49 ++++++++++++++++++ .../DataApiDropNamespaceOpDispenser.java | 50 +++++++++++++++++++ .../DataApiGetDatabaseInfoOpDispenser.java | 50 +++++++++++++++++++ .../DataApiListNamespacesOpDispenser.java | 48 ++++++++++++++++++ .../dataapi/ops/DataApiCreateNamespaceOp.java | 36 +++++++++++++ .../adapter/dataapi/ops/DataApiDBAdminOp.java | 33 ++++++++++++ .../dataapi/ops/DataApiDropNamespaceOp.java | 35 +++++++++++++ .../dataapi/ops/DataApiGetDatabaseInfoOp.java | 37 ++++++++++++++ .../dataapi/ops/DataApiListNamespacesOp.java | 32 ++++++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 4 ++ 12 files changed, 386 insertions(+) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateNamespaceOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropNamespaceOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiGetDatabaseInfoOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListNamespacesOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateNamespaceOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDBAdminOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropNamespaceOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiGetDatabaseInfoOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListNamespacesOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 69b5f59d7..5e5c5b4b7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -74,6 +74,10 @@ public class DataApiOpMapper implements OpMapper { case create_database -> new DataApiCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); case list_databases -> new DataApiListDatabasesOpDispenser(adapter, op, typeAndTarget.targetFunction); case drop_database -> new DataApiDropDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction); + case get_database_info -> new DataApiGetDatabaseInfoOpDispenser(adapter, op, typeAndTarget.targetFunction); + case create_namespace -> new DataApiCreateNamespaceOpDispenser(adapter, op, typeAndTarget.targetFunction); + case list_namespaces -> new DataApiListNamespacesOpDispenser(adapter, op, typeAndTarget.targetFunction); + case drop_namespace -> new DataApiDropNamespaceOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index 8602f3789..305ae9f4b 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.dataapi; import com.datastax.astra.client.DataAPIClient; import com.datastax.astra.client.Database; import com.datastax.astra.client.admin.AstraDBAdmin; +import com.datastax.astra.client.admin.DatabaseAdmin; import io.nosqlbench.nb.api.config.standard.ConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfiguration; @@ -32,6 +33,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; +import java.util.UUID; public class DataApiSpace { private final static Logger logger = LogManager.getLogger(DataApiSpace.class); @@ -45,6 +47,7 @@ public class DataApiSpace { private String superUserToken; private AstraDBAdmin admin; + private DatabaseAdmin namespaceAdmin; public DataApiSpace(String name, NBConfiguration cfg) { this.config = cfg; @@ -68,6 +71,10 @@ public class DataApiSpace { return admin; } + public DatabaseAdmin getNamespaceAdmin() { + return namespaceAdmin; + } + private void createClient() { this.dataAPIClient = new DataAPIClient(astraToken); if (namespace != null) { @@ -80,6 +87,7 @@ public class DataApiSpace { } else { this.admin = dataAPIClient.getAdmin(); } + this.namespaceAdmin = database.getDatabaseAdmin(); } private void setApiEndpoint() { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateNamespaceOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateNamespaceOpDispenser.java new file mode 100644 index 000000000..a90e6c716 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateNamespaceOpDispenser.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCreateNamespaceOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiCreateNamespaceOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCreateNamespaceOpDispenser.class); + private final LongFunction opFunction; + + public DataApiCreateNamespaceOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiCreateNamespaceOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getNamespaceAdmin(), + targetFunction.apply(l) + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropNamespaceOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropNamespaceOpDispenser.java new file mode 100644 index 000000000..dc3dbeb6b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropNamespaceOpDispenser.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiCreateNamespaceOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDropNamespaceOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiDropNamespaceOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDropNamespaceOpDispenser.class); + private final LongFunction opFunction; + + public DataApiDropNamespaceOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiDropNamespaceOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getNamespaceAdmin(), + targetFunction.apply(l) + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiGetDatabaseInfoOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiGetDatabaseInfoOpDispenser.java new file mode 100644 index 000000000..5c841712c --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiGetDatabaseInfoOpDispenser.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiGetDatabaseInfoOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.UUID; +import java.util.function.LongFunction; + +public class DataApiGetDatabaseInfoOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiGetDatabaseInfoOpDispenser.class); + private final LongFunction opFunction; + + public DataApiGetDatabaseInfoOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiGetDatabaseInfoOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getAdmin(), + UUID.fromString(targetFunction.apply(l)) + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListNamespacesOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListNamespacesOpDispenser.java new file mode 100644 index 000000000..2b90216b7 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiListNamespacesOpDispenser.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiListNamespacesOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiListNamespacesOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiListNamespacesOpDispenser.class); + private final LongFunction opFunction; + + public DataApiListNamespacesOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiListNamespacesOp( + spaceFunction.apply(l).getDatabase(), + spaceFunction.apply(l).getNamespaceAdmin() + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateNamespaceOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateNamespaceOp.java new file mode 100644 index 000000000..4b6e1f19b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateNamespaceOp.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; +import com.datastax.astra.client.admin.DatabaseAdmin; + +public class DataApiCreateNamespaceOp extends DataApiDBAdminOp { + private final String namespace; + + public DataApiCreateNamespaceOp(Database db, DatabaseAdmin admin, String namespace) { + super(db, admin); + this.namespace = namespace; + } + + @Override + public Object apply(long value) { + admin.createNamespace(namespace); + return true; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDBAdminOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDBAdminOp.java new file mode 100644 index 000000000..d8d750e13 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDBAdminOp.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; +import com.datastax.astra.client.admin.DatabaseAdmin; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public abstract class DataApiDBAdminOp extends DataApiBaseOp { + protected static final Logger logger = LogManager.getLogger(DataApiDBAdminOp.class); + protected final DatabaseAdmin admin; + + public DataApiDBAdminOp(Database db, DatabaseAdmin admin) { + super(db); + this.admin = admin; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropNamespaceOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropNamespaceOp.java new file mode 100644 index 000000000..a79a4c5af --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropNamespaceOp.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.DatabaseAdmin; + +public class DataApiDropNamespaceOp extends DataApiDBAdminOp { + private final String namespace; + + public DataApiDropNamespaceOp(Database db, DatabaseAdmin admin, String namespace) { + super(db, admin); + this.namespace = namespace; + } + + @Override + public Object apply(long value) { + admin.dropNamespace(namespace); + return true; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiGetDatabaseInfoOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiGetDatabaseInfoOp.java new file mode 100644 index 000000000..bebacfaa8 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiGetDatabaseInfoOp.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.AstraDBAdmin; + +import java.util.UUID; +import java.util.regex.Pattern; + +public class DataApiGetDatabaseInfoOp extends DataApiAdminOp { + private final UUID uuid; + + public DataApiGetDatabaseInfoOp(Database db, AstraDBAdmin admin, UUID uuid) { + super(db, admin); + this.uuid = uuid; + } + + @Override + public Object apply(long value) { + return admin.getDatabaseInfo(uuid); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListNamespacesOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListNamespacesOp.java new file mode 100644 index 000000000..8afcb2ff5 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiListNamespacesOp.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.admin.DatabaseAdmin; + +public class DataApiListNamespacesOp extends DataApiDBAdminOp { + + public DataApiListNamespacesOp(Database db, DatabaseAdmin admin) { + super(db, admin); + } + + @Override + public Object apply(long value) { + return admin.listNamespaceNames(); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index 634116e8e..e779d250d 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -45,4 +45,8 @@ public enum DataApiOpType { create_database, list_databases, drop_database, + get_database_info, + create_namespace, + list_namespaces, + drop_namespace, } From 12d4c8f6d689c7e8b476f9006b236d3916389dd7 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Mon, 17 Jun 2024 16:27:01 -0400 Subject: [PATCH 13/19] fixed issue with new parameters --- .../java/io/nosqlbench/adapter/dataapi/DataApiSpace.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index 305ae9f4b..c9953891c 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -33,7 +33,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; -import java.util.UUID; public class DataApiSpace { private final static Logger logger = LogManager.getLogger(DataApiSpace.class); @@ -173,6 +172,14 @@ public class DataApiSpace { .setDescription("The Astra namespace to use") ) + .add( + Param.optional("superTokenFile", String.class) + .setDescription("optional file to load Astra admin user token from") + ) + .add( + Param.optional("superToken", String.class) + .setDescription("optional Astra token used to connect as Admin user") + ) .asReadOnly(); } From 22bfffc0eb65bee7a782d660e316decc306f8e75 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 17 Jun 2024 17:38:41 -0500 Subject: [PATCH 14/19] cancel update timer on complete --- .../java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java index b4e98b7d1..6eb2781d1 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java @@ -97,6 +97,9 @@ public class ResolverForNBIOCache implements ContentResolver { public void run() { double progress = (double) totalBytesRead / fileSize * 100; logger.info(() -> "Progress: " + String.format("%.2f", progress) + "% completed"); + if (totalBytesRead==fileSize) { + cancel(); + } } } From f1c8d3ad245c01d766566177335bd1e6be2a5e0b Mon Sep 17 00:00:00 2001 From: Christopher Bradford Date: Tue, 18 Jun 2024 17:18:55 -0400 Subject: [PATCH 15/19] Update baseline cql_vector.yaml --- .../src/main/resources/activities/baselines/cql_vector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml b/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml index 8a1eb9eb6..c166efe10 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml +++ b/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml @@ -17,7 +17,7 @@ scenarios: default: schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto - main_write: run driver=cql tags==block:main-write cycles===TEMPLATE(main-cycles,10000000) threads=auto + main_write: run driver=cql tags==block:main_write cycles===TEMPLATE(main-cycles,10000000) threads=auto main_read_ann: run driver=cql tags==block:main_read_ann cycles===TEMPLATE(main-cycles,10000000) threads=auto main_read_pk_ann: run driver=cql tags==block:main_read_pk_ann cycles===TEMPLATE(main-cycles,10000000) threads=auto main_read: run driver=cql tags==block:"main-read.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto From 79c376b687b616563a957df23cb2be886cfb7b6a Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 18 Jun 2024 17:14:53 -0500 Subject: [PATCH 16/19] enable nbio cache by default --- .../main/java/io/nosqlbench/engine/cli/NBCLIOptions.java | 8 ++++---- nb-engine/nb-engine-cli/src/main/resources/nbiocache.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index dd2970ee4..8708c4630 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -137,7 +137,7 @@ public class NBCLIOptions { private static final String DEFAULT_CONSOLE_PATTERN = "TERSE"; private static final String DEFAULT_LOGFILE_PATTERN = "VERBOSE"; private final static String ENABLE_DEDICATED_VERIFICATION_LOGGER = "--enable-dedicated-verification-logging"; - private final static String USE_NBIO_CACHE = "--use-nbio-cache"; + private final static String DISABLE_NBIO_CACHE = "--disable-nbio-cache"; private final static String NBIO_CACHE_FORCE_UPDATE = "--nbio-cache-force-update"; private final static String NBIO_CACHE_NO_VERIFY = "--nbio-cache-no-verify"; private final static String NBIO_CACHE_DIR = "--nbio-cache-dir"; @@ -211,7 +211,7 @@ public class NBCLIOptions { private String metricsLabelSpec = ""; private String wantsToCatResource = ""; private long heartbeatIntervalMs = 10000; - private boolean useNBIOCache = false; + private boolean useNBIOCache = true; private boolean nbioCacheForceUpdate = false; private boolean nbioCacheVerify = true; private String nbioCacheDir; @@ -661,9 +661,9 @@ public class NBCLIOptions { this.heartbeatIntervalMs = Long.parseLong(this.readWordOrThrow(arglist, "heartbeat interval in ms")); break; - case USE_NBIO_CACHE: + case DISABLE_NBIO_CACHE: arglist.removeFirst(); - this.useNBIOCache = true; + this.useNBIOCache = false; break; case NBIO_CACHE_FORCE_UPDATE: arglist.removeFirst(); diff --git a/nb-engine/nb-engine-cli/src/main/resources/nbiocache.md b/nb-engine/nb-engine-cli/src/main/resources/nbiocache.md index 796a0347b..67f3c01a9 100644 --- a/nb-engine/nb-engine-cli/src/main/resources/nbiocache.md +++ b/nb-engine/nb-engine-cli/src/main/resources/nbiocache.md @@ -7,11 +7,11 @@ The following options are available via the command line to configure this featu ## Caching Options -Enable caching. Note that by default caching is disabled. It may be turned on with the following parameter +Disable caching. Note that by default caching is enabled. It may be turned off with the following parameter - --use-nbio-cache + --disable-nbio-cache -If caching is enabled by default nb will use the directory `$HOME/.nosqlbench/nbiocache` to store files. +If caching is enabled, nb will use the directory `$HOME/.nosqlbench/nbiocache` to store files. This location can be user specified with the parameter --nbio-cache-dir From 72fd68d69e133d0efb060294f1a67bf03a24d770 Mon Sep 17 00:00:00 2001 From: Christopher Bradford Date: Tue, 18 Jun 2024 23:08:05 -0400 Subject: [PATCH 17/19] Update cql_vector.yaml Fixed issues replacing dashes with underscores --- .../resources/activities/baselines/cql_vector.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml b/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml index c166efe10..c3eb262ce 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml +++ b/nb-adapters/adapter-cqld4/src/main/resources/activities/baselines/cql_vector.yaml @@ -20,13 +20,13 @@ scenarios: main_write: run driver=cql tags==block:main_write cycles===TEMPLATE(main-cycles,10000000) threads=auto main_read_ann: run driver=cql tags==block:main_read_ann cycles===TEMPLATE(main-cycles,10000000) threads=auto main_read_pk_ann: run driver=cql tags==block:main_read_pk_ann cycles===TEMPLATE(main-cycles,10000000) threads=auto - main_read: run driver=cql tags==block:"main-read.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto + main_read: run driver=cql tags==block:"main_read.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto main: run driver=cql tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto astra: - schema: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF - drop_tables: run driver=cql tags==block:drop-tables threads==1 cycles==UNDEF - truncate: run driver=cql tags==block:truncate-tables cycles===1 threads=1 - reads: run driver=cql tags==block:main-read cycles===TEMPLATE(main-cycles,10000000) threads=auto + schema: run driver=cql tags==block:schema_astra threads==1 cycles==UNDEF + drop_tables: run driver=cql tags==block:drop_tables threads==1 cycles==UNDEF + truncate: run driver=cql tags==block:truncate_tables cycles===1 threads=1 + reads: run driver=cql tags==block:"main_read.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto bindings: seq_key: Mod(<>); ToString() -> String From 883cde6fae0fc27b97e45ec4b9045b7100ce5645 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 24 Jun 2024 11:56:27 -0500 Subject: [PATCH 18/19] remove extraneous run config --- .run/linkedinput.run.xml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .run/linkedinput.run.xml diff --git a/.run/linkedinput.run.xml b/.run/linkedinput.run.xml deleted file mode 100644 index 4e79f005b..000000000 --- a/.run/linkedinput.run.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - \ No newline at end of file From 014c82dc518aa9a294e2c403914f8ba7648e5a56 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 24 Jun 2024 11:56:39 -0500 Subject: [PATCH 19/19] nosqlbench-1966 ArrayIndexOutOfBoundsException in ArgsComparator during resolveFunctions call --- .../engine/core/lifecycle/session/NBSession.java | 2 +- .../virtdata/core/bindings/ArgsComparator.java | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java index f702d0ef3..4bc14fd47 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java @@ -89,7 +89,7 @@ public class NBSession extends NBHeartbeatComponent implements Function> { int paramLen = ctor.getParameterCount(); int argsLen = arguments.length; - if (paramLen!=argsLen && !ctor.isVarArgs()) { - return MATCHRANK.INCOMPATIBLE; + if (ctor.isVarArgs()) { + if (argsLen> { Class[] atypes = Arrays.stream(arguments).map(Object::getClass).toArray(i -> new Class[i]); for (int position = 0; position < len; position++) { - Class ptype = ptypes[position]; + + Class ptype = (position atype = (position across = WRAPPER_TYPE_MAP.get(atype); Class pcross = WRAPPER_TYPE_MAP.get(ptype);