From 101fc5b15c290762ad9345a08589671375e48e1e Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Tue, 7 May 2024 23:48:28 -0400 Subject: [PATCH 01/19] initial checkin --- nb-adapters/adapter-dataapi/README.md | 0 nb-adapters/adapter-dataapi/pom.xml | 57 +++++++++++ .../adapter/dataapi/DataApiDriverAdapter.java | 51 ++++++++++ .../dataapi/DataApiDriverAdapterLoader.java | 30 ++++++ .../adapter/dataapi/DataApiOpMapper.java | 61 ++++++++++++ .../adapter/dataapi/DataApiSpace.java | 99 +++++++++++++++++++ .../opdispensers/DataApiBaseOpDispenser.java | 31 ++++++ .../DataApiCreateCollectionOpDispenser.java | 61 ++++++++++++ .../DataApiDeleteCollectionOpDispenser.java | 34 +++++++ .../DataApiDeleteManyOpDispenser.java | 34 +++++++ .../DataApiDeleteOneOpDispenser.java | 34 +++++++ .../DataApiFindOneAndDeleteOpDispenser.java | 34 +++++++ .../DataApiFindOneAndUpdateOpDispenser.java | 34 +++++++ .../DataApiFindOneOpDispenser.java | 34 +++++++ .../opdispensers/DataApiFindOpDispenser.java | 34 +++++++ .../DataApiInsertManyOpDispenser.java | 34 +++++++ .../DataApiInsertOneOpDispenser.java | 34 +++++++ .../DataApiUpdateManyOpDispenser.java | 34 +++++++ .../DataApiUpdateOneOpDispenser.java | 34 +++++++ .../adapter/dataapi/ops/DataApiBaseOp.java | 22 +++++ .../adapter/dataapi/ops/DataApiOpType.java | 16 +++ nb-adapters/nb-adapters-included/pom.xml | 12 ++- nb-adapters/pom.xml | 12 ++- 23 files changed, 824 insertions(+), 2 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/README.md create mode 100644 nb-adapters/adapter-dataapi/pom.xml create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapterLoader.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java diff --git a/nb-adapters/adapter-dataapi/README.md b/nb-adapters/adapter-dataapi/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/pom.xml b/nb-adapters/adapter-dataapi/pom.xml new file mode 100644 index 000000000..8cd1a0691 --- /dev/null +++ b/nb-adapters/adapter-dataapi/pom.xml @@ -0,0 +1,57 @@ + + + +4.0.0 + +adapter-dataapi +jar + + + mvn-defaults + io.nosqlbench + ${revision} + ../../mvn-defaults + + +${project.artifactId} + + An nosqlbench adapter driver module for the DataStax Data API + + + + + + io.nosqlbench + nb-annotations + ${revision} + compile + + + io.nosqlbench + adapters-api + ${revision} + compile + + + com.datastax.astra + astra-db-java + 1.0.0 + + + + + diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java new file mode 100644 index 000000000..0a4f7f34a --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi; + +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.labels.NBLabels; + +import java.util.function.Function; + +@Service(value = DriverAdapter.class, selector = "dataapi") +public class DataApiDriverAdapter extends BaseDriverAdapter, DataApiSpace> { + public DataApiDriverAdapter(NBComponent parent, NBLabels childLabels) { + super(parent, childLabels); + } + + @Override + public OpMapper> getOpMapper() { + return new DataApiOpMapper(this); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return (s) -> new DataApiSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(DataApiSpace.getConfigModel()); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapterLoader.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapterLoader.java new file mode 100644 index 000000000..71f4e8049 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapterLoader.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi; + +import io.nosqlbench.adapter.diag.DriverAdapterLoader; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.labels.NBLabels; + +@Service(value = DriverAdapterLoader.class, selector = "dataapi") +public class DataApiDriverAdapterLoader implements DriverAdapterLoader { + @Override + public DataApiDriverAdapter load(NBComponent parent, NBLabels childLabels) { + return new DataApiDriverAdapter(parent, childLabels); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java new file mode 100644 index 000000000..8e5d38c99 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi; + +import io.nosqlbench.adapter.dataapi.opdispensers.*; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiOpType; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.engine.api.templating.TypeAndTarget; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DataApiOpMapper implements OpMapper> { + private static final Logger logger = LogManager.getLogger(DataApiOpMapper.class); + private final DataApiDriverAdapter adapter; + + public DataApiOpMapper(DataApiDriverAdapter dataApiDriverAdapter) { + this.adapter = dataApiDriverAdapter; + } + + @Override + public OpDispenser> apply(ParsedOp op) { + TypeAndTarget typeAndTarget = op.getTypeAndTarget( + DataApiOpType.class, + String.class, + "type", + "target" + ); + logger.debug(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); + return switch (typeAndTarget.enumId) { + case create_collection -> new DataApiCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert_many -> new DataApiInsertManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert_one -> new DataApiInsertOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find -> new DataApiFindOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one -> new DataApiFindOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one_and_delete -> new DataApiFindOneAndDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_one_and_update -> new DataApiFindOneAndUpdateOpDispenser(adapter, op, typeAndTarget.targetFunction); + case update_one -> new DataApiUpdateOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case update_many -> new DataApiUpdateManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_one -> new DataApiDeleteOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_many -> new DataApiDeleteManyOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_collection -> new DataApiDeleteCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + }; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java new file mode 100644 index 000000000..52de33322 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi; + +import com.datastax.astra.client.DataAPIClient; +import com.datastax.astra.client.Database; +import io.nosqlbench.nb.api.config.standard.ConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.config.standard.Param; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; + +public class DataApiSpace { + private final static Logger logger = LogManager.getLogger(DataApiSpace.class); + private final NBConfiguration config; + private final String name; + private String astraToken; + private String astraApiEndpoint; + private DataAPIClient dataAPIClient; + private Database database; + public DataApiSpace(String name, NBConfiguration cfg) { + this.config = cfg; + this.name = name; + setToken(); + setApiEndpoint(); + createClient(); + } + + public DataAPIClient getDataAPIClient() { + return dataAPIClient; + } + + public Database getDatabase() { + return database; + } + + private void createClient() { + this.dataAPIClient = new DataAPIClient(astraToken); + this.database = dataAPIClient.getDatabase(astraApiEndpoint); + } + + private void setApiEndpoint() { + this.astraApiEndpoint = config.get("astraApiEndpoint"); + } + + private void setToken() { + String tokenFileContents = null; + Optional tokenFilePath = config.getOptional("astraTokenFile"); + if(tokenFilePath.isPresent()) { + Path path = Paths.get(tokenFilePath.get()); + try { + tokenFileContents = Files.readAllLines(path).getFirst(); + } catch (IOException e) { + String error = "Error while reading token from file:" + path; + logger.error(error, e); + throw new RuntimeException(e); + } + } + this.astraToken = (tokenFileContents != null) ? tokenFileContents : config.get("astraToken"); + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(DataApiSpace.class) + .add( + Param.optional("astraTokenFile", String.class, "file to load the Astra token from") + ) + .add( + Param.optional("astraToken",String.class) + .setDescription("the Astra token used to connect to the database") + ) + .add( + Param.defaultTo("astraApiEndpoint", String.class) + .setDescription("the API endpoint for the Astra database") + ) + .asReadOnly(); + } + +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java new file mode 100644 index 000000000..ef30a1450 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiSpace; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +public abstract class DataApiBaseOpDispenser extends BaseOpDispenser, DataApiSpace> { + + protected DataApiBaseOpDispenser(DriverAdapter,? extends DataApiSpace> adapter, ParsedOp op) { + super(adapter, op); + } + +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java new file mode 100644 index 000000000..c7f21c482 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.evalctx.CycleFunction; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiCreateCollectionOpDispenser extends DataApiBaseOpDispenser { + public DataApiCreateCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp getOp(long value) { + return null; + } + + @Override + public CycleFunction getVerifier() { + return null; + } + + @Override + public String getOpName() { + return null; + } + + @Override + public void onStart(long cycleValue) { + + } + + @Override + public void onSuccess(long cycleValue, long nanoTime) { + + } + + @Override + public void onError(long cycleValue, long resultNanos, Throwable t) { + + } + +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java new file mode 100644 index 000000000..48468c997 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiDeleteCollectionOpDispenser extends DataApiBaseOpDispenser { + public DataApiDeleteCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java new file mode 100644 index 000000000..76893131c --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiDeleteManyOpDispenser extends DataApiBaseOpDispenser { + public DataApiDeleteManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java new file mode 100644 index 000000000..06e7fce84 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiDeleteOneOpDispenser extends DataApiBaseOpDispenser { + public DataApiDeleteOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java new file mode 100644 index 000000000..69bb0be23 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiFindOneAndDeleteOpDispenser extends DataApiBaseOpDispenser { + public DataApiFindOneAndDeleteOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java new file mode 100644 index 000000000..c07d846d8 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiFindOneAndUpdateOpDispenser extends DataApiBaseOpDispenser { + public DataApiFindOneAndUpdateOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java new file mode 100644 index 000000000..1076f1275 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiFindOneOpDispenser extends DataApiBaseOpDispenser { + public DataApiFindOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java new file mode 100644 index 000000000..3e2a98629 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiFindOpDispenser extends DataApiBaseOpDispenser { + public DataApiFindOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java new file mode 100644 index 000000000..685b35c5d --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiInsertManyOpDispenser extends DataApiBaseOpDispenser { + public DataApiInsertManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java new file mode 100644 index 000000000..8db5982cd --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiInsertOneOpDispenser extends DataApiBaseOpDispenser { + public DataApiInsertOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java new file mode 100644 index 000000000..a375e1b02 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiUpdateManyOpDispenser extends DataApiBaseOpDispenser { + public DataApiUpdateManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java new file mode 100644 index 000000000..2b400eb74 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class DataApiUpdateOneOpDispenser extends DataApiBaseOpDispenser { + public DataApiUpdateOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } + + @Override + public DataApiBaseOp> getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java new file mode 100644 index 000000000..84074abd3 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; + +public class DataApiBaseOp implements Op { +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java new file mode 100644 index 000000000..296efe2ed --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -0,0 +1,16 @@ +package io.nosqlbench.adapter.dataapi.ops; + +public enum DataApiOpType { + create_collection, + insert_many, + insert_one, + find, + find_one, + find_one_and_delete, + find_one_and_update, + update_one, + update_many, + delete_one, + delete_many, + delete_collection +} diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 5c108177c..3721199c0 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -186,7 +186,7 @@ adapter-s4j-include - true + false @@ -238,6 +238,16 @@ + + + + + + + + + + diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index 08af473e8..97a84dfb5 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -147,7 +147,7 @@ adapter-s4j-module - true + false adapter-s4j @@ -174,5 +174,15 @@ + + adapter-dataapi + + true + + + adapter-dataapi + + + From f5b9660b315126686e5b37035f6e48a7dae7abc5 Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Tue, 7 May 2024 23:54:58 -0400 Subject: [PATCH 02/19] add copyright --- .../adapter/dataapi/ops/DataApiOpType.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index 296efe2ed..c2f40fe2a 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; public enum DataApiOpType { From a76f69d227cecb63d7647707e304495b96cb953a Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Wed, 8 May 2024 12:55:29 -0400 Subject: [PATCH 03/19] adding op functionality --- .../adapter/dataapi/DataApiDriverAdapter.java | 4 +- .../adapter/dataapi/DataApiOpMapper.java | 6 +- .../adapter/dataapi/DataApiSpace.java | 5 ++ .../DataApiCreateCollectionOpDispenser.java | 70 +++++++++++-------- .../DataApiDeleteManyOpDispenser.java | 6 +- .../DataApiDeleteOneOpDispenser.java | 6 +- .../DataApiDropCollectionOpDispenser.java | 47 +++++++++++++ .../DataApiFindOneAndDeleteOpDispenser.java | 6 +- .../DataApiFindOneAndUpdateOpDispenser.java | 6 +- .../DataApiFindOneOpDispenser.java | 6 +- .../opdispensers/DataApiFindOpDispenser.java | 6 +- .../DataApiInsertManyOpDispenser.java | 6 +- .../DataApiInsertOneOpDispenser.java | 6 +- ...Dispenser.java => DataApiOpDispenser.java} | 11 ++- .../DataApiUpdateManyOpDispenser.java | 6 +- .../DataApiUpdateOneOpDispenser.java | 6 +- .../adapter/dataapi/ops/DataApiBaseOp.java | 10 ++- .../DataApiCreateCollectionOp.java} | 22 +++--- .../dataapi/ops/DataApiDropCollectionOp.java | 20 ++++++ 19 files changed, 177 insertions(+), 78 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java rename nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/{DataApiBaseOpDispenser.java => DataApiOpDispenser.java} (64%) rename nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/{opdispensers/DataApiDeleteCollectionOpDispenser.java => ops/DataApiCreateCollectionOp.java} (51%) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java index 0a4f7f34a..8c927784e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiDriverAdapter.java @@ -29,13 +29,13 @@ import io.nosqlbench.nb.api.labels.NBLabels; import java.util.function.Function; @Service(value = DriverAdapter.class, selector = "dataapi") -public class DataApiDriverAdapter extends BaseDriverAdapter, DataApiSpace> { +public class DataApiDriverAdapter extends BaseDriverAdapter { public DataApiDriverAdapter(NBComponent parent, NBLabels childLabels) { super(parent, childLabels); } @Override - public OpMapper> getOpMapper() { + public OpMapper getOpMapper() { return new DataApiOpMapper(this); } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 8e5d38c99..7e1ba4e1a 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -26,7 +26,7 @@ import io.nosqlbench.engine.api.templating.TypeAndTarget; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class DataApiOpMapper implements OpMapper> { +public class DataApiOpMapper implements OpMapper { private static final Logger logger = LogManager.getLogger(DataApiOpMapper.class); private final DataApiDriverAdapter adapter; @@ -35,7 +35,7 @@ public class DataApiOpMapper implements OpMapper> { } @Override - public OpDispenser> apply(ParsedOp op) { + public OpDispenser apply(ParsedOp op) { TypeAndTarget typeAndTarget = op.getTypeAndTarget( DataApiOpType.class, String.class, @@ -55,7 +55,7 @@ public class DataApiOpMapper implements OpMapper> { case update_many -> new DataApiUpdateManyOpDispenser(adapter, op, typeAndTarget.targetFunction); case delete_one -> new DataApiDeleteOneOpDispenser(adapter, op, typeAndTarget.targetFunction); case delete_many -> new DataApiDeleteManyOpDispenser(adapter, op, typeAndTarget.targetFunction); - case delete_collection -> new DataApiDeleteCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + case delete_collection -> new DataApiDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index 52de33322..0edf579e7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.dataapi; +import com.datastax.astra.client.Collection; import com.datastax.astra.client.DataAPIClient; import com.datastax.astra.client.Database; import io.nosqlbench.nb.api.config.standard.ConfigModel; @@ -58,6 +59,10 @@ public class DataApiSpace { private void createClient() { this.dataAPIClient = new DataAPIClient(astraToken); this.database = dataAPIClient.getDatabase(astraApiEndpoint); +// database.getCollection("test"); +// database.listCollections().forEach(System.out::println); +// Collection collection = database.getCollection("test"); +// collection.deleteMany() } private void setApiEndpoint() { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java index c7f21c482..5371196d4 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java @@ -16,46 +16,58 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.model.CollectionOptions; +import com.datastax.astra.client.model.SimilarityMetric; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapters.api.evalctx.CycleFunction; +import io.nosqlbench.adapter.dataapi.ops.DataApiCreateCollectionOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Optional; import java.util.function.LongFunction; -public class DataApiCreateCollectionOpDispenser extends DataApiBaseOpDispenser { +public class DataApiCreateCollectionOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiCreateCollectionOpDispenser.class); + private final LongFunction opFunction; + public DataApiCreateCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + CollectionOptions.CollectionOptionsBuilder optionsBldr = CollectionOptions.builder(); + Optional> dimFunc = op.getAsOptionalFunction("dimensions", Integer.class); + if (dimFunc.isPresent()) { + LongFunction af = dimFunc.get(); + optionsBldr.vectorDimension(af.apply(l)); + } +// COSINE("cosine"), +// EUCLIDEAN("euclidean"), +// DOT_PRODUCT("dot_product"); + Optional> simFunc = op.getAsOptionalFunction("similarity", String.class); + if (simFunc.isPresent()) { + LongFunction sf = simFunc.get(); + optionsBldr.vectorSimilarity(SimilarityMetric.valueOf(sf.apply(l))); + } + + DataApiCreateCollectionOp dataApiCreateCollectionOp = + new DataApiCreateCollectionOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + optionsBldr.build()); + + return dataApiCreateCollectionOp; + }; } @Override - public DataApiBaseOp getOp(long value) { - return null; + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); } - @Override - public CycleFunction getVerifier() { - return null; - } - - @Override - public String getOpName() { - return null; - } - - @Override - public void onStart(long cycleValue) { - - } - - @Override - public void onSuccess(long cycleValue, long nanoTime) { - - } - - @Override - public void onError(long cycleValue, long resultNanos, Throwable t) { - - } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java index 76893131c..674e8b706 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiDeleteManyOpDispenser extends DataApiBaseOpDispenser { +public class DataApiDeleteManyOpDispenser extends DataApiOpDispenser { public DataApiDeleteManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java index 06e7fce84..eb3c01fff 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiDeleteOneOpDispenser extends DataApiBaseOpDispenser { +public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser { public DataApiDeleteOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java new file mode 100644 index 000000000..dd3748956 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDropCollectionOpDispenser.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDropCollectionOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiDropCollectionOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDropCollectionOpDispenser.class); + private final LongFunction opFunction; + public DataApiDropCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new DataApiDropCollectionOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l) + ); + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java index 69bb0be23..a4f8cfab7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiFindOneAndDeleteOpDispenser extends DataApiBaseOpDispenser { +public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser { public DataApiFindOneAndDeleteOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java index c07d846d8..7f5d6fb03 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiFindOneAndUpdateOpDispenser extends DataApiBaseOpDispenser { +public class DataApiFindOneAndUpdateOpDispenser extends DataApiOpDispenser { public DataApiFindOneAndUpdateOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java index 1076f1275..3b6c6090e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiFindOneOpDispenser extends DataApiBaseOpDispenser { +public class DataApiFindOneOpDispenser extends DataApiOpDispenser { public DataApiFindOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index 3e2a98629..6d6a0fe24 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiFindOpDispenser extends DataApiBaseOpDispenser { +public class DataApiFindOpDispenser extends DataApiOpDispenser { public DataApiFindOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java index 685b35c5d..25b2a6537 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiInsertManyOpDispenser extends DataApiBaseOpDispenser { +public class DataApiInsertManyOpDispenser extends DataApiOpDispenser { public DataApiInsertManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java index 8db5982cd..ee5b118a6 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiInsertOneOpDispenser extends DataApiBaseOpDispenser { +public class DataApiInsertOneOpDispenser extends DataApiOpDispenser { public DataApiInsertOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java similarity index 64% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index ef30a1450..45f5199af 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiBaseOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -22,10 +22,17 @@ import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; -public abstract class DataApiBaseOpDispenser extends BaseOpDispenser, DataApiSpace> { +import java.util.function.LongFunction; - protected DataApiBaseOpDispenser(DriverAdapter,? extends DataApiSpace> adapter, ParsedOp op) { +public abstract class DataApiOpDispenser extends BaseOpDispenser { + protected final LongFunction targetFunction; + protected final LongFunction spaceFunction; + + protected DataApiOpDispenser(DriverAdapter adapter, ParsedOp op, + LongFunction targetFunction) { super(adapter, op); + this.targetFunction = targetFunction; + this.spaceFunction = adapter.getSpaceFunc(op); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java index a375e1b02..82269b156 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateManyOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiUpdateManyOpDispenser extends DataApiBaseOpDispenser { +public class DataApiUpdateManyOpDispenser extends DataApiOpDispenser { public DataApiUpdateManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java index 2b400eb74..114c7945e 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiUpdateOneOpDispenser.java @@ -22,13 +22,13 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; -public class DataApiUpdateOneOpDispenser extends DataApiBaseOpDispenser { +public class DataApiUpdateOneOpDispenser extends DataApiOpDispenser { public DataApiUpdateOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); } @Override - public DataApiBaseOp> getOp(long value) { + public DataApiBaseOp getOp(long value) { return null; } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java index 84074abd3..13181c5db 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiBaseOp.java @@ -16,7 +16,13 @@ package io.nosqlbench.adapter.dataapi.ops; -import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import com.datastax.astra.client.Database; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; -public class DataApiBaseOp implements Op { +public abstract class DataApiBaseOp implements CycleOp { + protected final Database db; + + public DataApiBaseOp(Database db) { + this.db = db; + } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionOp.java similarity index 51% rename from nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java rename to nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionOp.java index 48468c997..625ca385a 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiCreateCollectionOp.java @@ -14,21 +14,23 @@ * limitations under the License. */ -package io.nosqlbench.adapter.dataapi.opdispensers; +package io.nosqlbench.adapter.dataapi.ops; -import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; -import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; -import io.nosqlbench.adapters.api.templating.ParsedOp; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.CollectionOptions; -import java.util.function.LongFunction; +public class DataApiCreateCollectionOp extends DataApiBaseOp { + private final String collectionName; + private final CollectionOptions options; -public class DataApiDeleteCollectionOpDispenser extends DataApiBaseOpDispenser { - public DataApiDeleteCollectionOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + public DataApiCreateCollectionOp(Database db, String collectionName, CollectionOptions options) { + super(db); + this.collectionName = collectionName; + this.options = options; } @Override - public DataApiBaseOp> getOp(long value) { - return null; + public Object apply(long value) { + return db.createCollection(collectionName, options); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java new file mode 100644 index 000000000..ba45f98a0 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java @@ -0,0 +1,20 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; + +public class DataApiDropCollectionOp extends DataApiBaseOp { + private final String collectionName; + public DataApiDropCollectionOp(Database db, String dbName) { + super(db); + this.collectionName = dbName; + } + + @Override + public Object apply(long value) { + Boolean exists = db.collectionExists(collectionName); + if (exists) { + db.dropCollection(collectionName); + } + return exists; + } +} From 0f6772ce3b57b17976ed3ceba3e5e20f083bc2fc Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Wed, 8 May 2024 17:58:23 -0400 Subject: [PATCH 04/19] checkpoint --- .../adapter/dataapi/DataApiSpace.java | 18 +++++--- .../DataApiCreateCollectionOpDispenser.java | 6 +-- .../DataApiDeleteManyOpDispenser.java | 24 ++++++++++- .../DataApiDeleteOneOpDispenser.java | 40 +++++++++++++++++- .../DataApiFindOneAndDeleteOpDispenser.java | 40 +++++++++++++++++- .../DataApiFindOneOpDispenser.java | 38 ++++++++++++++++- .../opdispensers/DataApiFindOpDispenser.java | 38 ++++++++++++++++- .../opdispensers/DataApiOpDispenser.java | 41 +++++++++++++++++++ .../dataapi/ops/DataApiDeleteManyOp.java | 21 ++++++++++ .../dataapi/ops/DataApiDeleteOneOp.java | 24 +++++++++++ .../ops/DataApiFindOneAndDeleteOp.java | 25 +++++++++++ .../adapter/dataapi/ops/DataApiFindOneOp.java | 25 +++++++++++ .../adapter/dataapi/ops/DataApiFindOp.java | 26 ++++++++++++ 13 files changed, 352 insertions(+), 14 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index 0edf579e7..ea0c5e1e9 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -16,7 +16,6 @@ package io.nosqlbench.adapter.dataapi; -import com.datastax.astra.client.Collection; import com.datastax.astra.client.DataAPIClient; import com.datastax.astra.client.Database; import io.nosqlbench.nb.api.config.standard.ConfigModel; @@ -40,11 +39,13 @@ public class DataApiSpace { private String astraApiEndpoint; private DataAPIClient dataAPIClient; private Database database; + private String namespace; public DataApiSpace(String name, NBConfiguration cfg) { this.config = cfg; this.name = name; setToken(); setApiEndpoint(); + setNamespace(); createClient(); } @@ -58,17 +59,22 @@ public class DataApiSpace { private void createClient() { this.dataAPIClient = new DataAPIClient(astraToken); - this.database = dataAPIClient.getDatabase(astraApiEndpoint); -// database.getCollection("test"); -// database.listCollections().forEach(System.out::println); -// Collection collection = database.getCollection("test"); -// collection.deleteMany() + if (namespace != null) { + this.database = dataAPIClient.getDatabase(astraApiEndpoint, namespace); + } else { + this.database = dataAPIClient.getDatabase(astraApiEndpoint); + } } private void setApiEndpoint() { this.astraApiEndpoint = config.get("astraApiEndpoint"); } + private void setNamespace() { + Optional maybeNamespace = config.getOptional("namespace"); + maybeNamespace.ifPresent(s -> this.namespace = s); + } + private void setToken() { String tokenFileContents = null; Optional tokenFilePath = config.getOptional("astraTokenFile"); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java index 5371196d4..d20a5521b 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java @@ -45,9 +45,9 @@ public class DataApiCreateCollectionOpDispenser extends DataApiOpDispenser { LongFunction af = dimFunc.get(); optionsBldr.vectorDimension(af.apply(l)); } -// COSINE("cosine"), -// EUCLIDEAN("euclidean"), -// DOT_PRODUCT("dot_product"); +// COSINE("cosine"), +// EUCLIDEAN("euclidean"), +// DOT_PRODUCT("dot_product"); Optional> simFunc = op.getAsOptionalFunction("similarity", String.class); if (simFunc.isPresent()) { LongFunction sf = simFunc.get(); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java index 674e8b706..0c0ce0f56 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java @@ -16,19 +16,41 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteManyOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; public class DataApiDeleteManyOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDeleteManyOpDispenser.class); + private final LongFunction opFunction; + public DataApiDeleteManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + + return new DataApiDeleteManyOp( + db, + db.getCollection(targetFunction.apply(l)), + filter + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java index eb3c01fff..3473267a8 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java @@ -16,19 +16,57 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.DeleteOneOptions; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Filters; +import com.datastax.astra.client.model.Sorts; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteOneOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiDeleteOneOpDispenser.class); + private final LongFunction opFunction; + public DataApiDeleteOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + DeleteOneOptions options = new DeleteOneOptions(); + Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); + if (sortFunction.isPresent()) { + Map sortFields = sortFunction.get().apply(l); + String sortOrder = sortFields.get("type").toString(); + String sortField = sortFields.get("field").toString(); + switch(sortOrder) { + case "asc" -> options = options.sort(Sorts.ascending(sortField)); + case "desc" -> options = options.sort(Sorts.descending(sortField)); + } + } + return new DataApiDeleteOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java index a4f8cfab7..47e594ae4 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java @@ -16,19 +16,57 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Filters; +import com.datastax.astra.client.model.FindOneAndDeleteOptions; +import com.datastax.astra.client.model.Sorts; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneAndDeleteOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindOneAndDeleteOpDispenser.class); + private final LongFunction opFunction; + public DataApiFindOneAndDeleteOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + FindOneAndDeleteOptions options = new FindOneAndDeleteOptions(); + Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); + if (sortFunction.isPresent()) { + Map sortFields = sortFunction.get().apply(l); + String sortOrder = sortFields.get("type").toString(); + String sortField = sortFields.get("field").toString(); + switch(sortOrder) { + case "asc" -> options = options.sort(Sorts.ascending(sortField)); + case "desc" -> options = options.sort(Sorts.descending(sortField)); + } + } + return new DataApiFindOneAndDeleteOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java index 3b6c6090e..87478e90b 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -16,19 +16,55 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneOptions; +import com.datastax.astra.client.model.Sorts; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindOneOpDispenser.class); + private final LongFunction opFunction; public DataApiFindOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + FindOneOptions options = new FindOneOptions(); + Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); + if (sortFunction.isPresent()) { + Map sortFields = sortFunction.get().apply(l); + String sortOrder = sortFields.get("type").toString(); + String sortField = sortFields.get("field").toString(); + switch(sortOrder) { + case "asc" -> options = options.sort(Sorts.ascending(sortField)); + case "desc" -> options = options.sort(Sorts.descending(sortField)); + } + } + return new DataApiFindOneOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index 6d6a0fe24..adc2665f6 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -16,19 +16,55 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOptions; +import com.datastax.astra.client.model.Sorts; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindOpDispenser.class); + private final LongFunction opFunction; public DataApiFindOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + FindOptions options = new FindOptions(); + Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); + if (sortFunction.isPresent()) { + Map sortFields = sortFunction.get().apply(l); + String sortOrder = sortFields.get("type").toString(); + String sortField = sortFields.get("field").toString(); + switch(sortOrder) { + case "asc" -> options = options.sort(Sorts.ascending(sortField)); + case "desc" -> options = options.sort(Sorts.descending(sortField)); + } + } + return new DataApiFindOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + options + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index 45f5199af..7ab96ad52 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -16,12 +16,18 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Filters; import io.nosqlbench.adapter.dataapi.DataApiSpace; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public abstract class DataApiOpDispenser extends BaseOpDispenser { @@ -35,4 +41,39 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> filterFunction = op.getAsOptionalFunction("filters", Map.class); + if (filterFunction.isPresent()) { + Map filters = filterFunction.get().apply(l); + List filterList = new ArrayList<>(); + String conjunction = "and"; + for (Map.Entry entry : filters.entrySet()) { + if (entry.getKey().equalsIgnoreCase("filter")) { + Map filterFields = (Map) entry.getValue(); + switch (filterFields.get("operation").toString()) { + case "lt" -> + filterList.add(Filters.lt(filters.get("field").toString(), (long) filters.get("value"))); + case "gt" -> + filterList.add(Filters.gt(filters.get("field").toString(), (long) filters.get("value"))); + case "eq" -> filterList.add(Filters.eq(filters.get("field").toString(), filters.get("value"))); + default -> logger.error("Operation not supported"); + } + } else if (entry.getKey().equalsIgnoreCase("conjunction")) { + conjunction = (String) entry.getValue(); + } else { + logger.error("Filter " + entry.getKey() + " not supported"); + } + } + if (conjunction.equalsIgnoreCase("and")) { + filter = Filters.and(filterList); + } else if (conjunction.equalsIgnoreCase("or")) { + filter = Filters.or(filterList); + } else { + logger.error("Conjunction " + conjunction + " not supported"); + } + } + return filter; + } + } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java new file mode 100644 index 000000000..cb67ccef5 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java @@ -0,0 +1,21 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; + +public class DataApiDeleteManyOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + + public DataApiDeleteManyOp(Database db, Collection collection, Filter filter) { + super(db); + this.collection = collection; + this.filter = filter; + } + + @Override + public Object apply(long value) { + return collection.deleteMany(filter); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java new file mode 100644 index 000000000..1e207d651 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java @@ -0,0 +1,24 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.DeleteOneOptions; +import com.datastax.astra.client.model.Filter; + +public class DataApiDeleteOneOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final DeleteOneOptions options; + + public DataApiDeleteOneOp(Database db, Collection collection, Filter filter, DeleteOneOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.deleteOne(filter, options); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java new file mode 100644 index 000000000..5123527d1 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java @@ -0,0 +1,25 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneAndDeleteOptions; + +public class DataApiFindOneAndDeleteOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final FindOneAndDeleteOptions options; + + public DataApiFindOneAndDeleteOp(Database db, Collection collection, Filter filter, FindOneAndDeleteOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.findOneAndDelete(filter, options); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java new file mode 100644 index 000000000..4ec93456d --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java @@ -0,0 +1,25 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneOptions; + +public class DataApiFindOneOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final FindOneOptions options; + + public DataApiFindOneOp(Database db, Collection collection, Filter filter, FindOneOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.findOne(filter, options); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java new file mode 100644 index 000000000..cfacd54ae --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java @@ -0,0 +1,26 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOneOptions; +import com.datastax.astra.client.model.FindOptions; + +public class DataApiFindOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final FindOptions options; + + public DataApiFindOp(Database db, Collection collection, Filter filter, FindOptions options) { + super(db); + this.collection = collection; + this.filter = filter; + this.options = options; + } + + @Override + public Object apply(long value) { + return collection.find(filter, options); + } +} From f7ea27ae5eec6c2b3d61f7ac2495375d29a333a8 Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Wed, 8 May 2024 18:24:43 -0400 Subject: [PATCH 05/19] adding insert ops --- .../DataApiInsertOneOpDispenser.java | 21 ++++++- .../DataApiInsertOneVectorOpDispenser.java | 56 +++++++++++++++++++ .../dataapi/ops/DataApiInsertOneOp.java | 20 +++++++ .../dataapi/ops/DataApiInsertOneVectorOp.java | 22 ++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java index ee5b118a6..2c43ca8ee 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java @@ -16,19 +16,38 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.model.Document; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiInsertOneOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; public class DataApiInsertOneOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiInsertOneOpDispenser.class); + private final LongFunction opFunction; + public DataApiInsertOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Document.parse(op.get("document", l)); + return new DataApiInsertOneOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + Document.parse(op.get("document", l)) + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java new file mode 100644 index 000000000..040dcbc01 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.model.Document; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiInsertOneOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiInsertOneVectorOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiInsertOneVectorOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiInsertOneVectorOpDispenser.class); + private final LongFunction opFunction; + + public DataApiInsertOneVectorOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Document.parse(op.get("document", l)); + float[] vector = op.get("vector", l); + return new DataApiInsertOneVectorOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + Document.parse(op.get("document", l)), + vector + ); + }; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java new file mode 100644 index 000000000..13fc5893b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java @@ -0,0 +1,20 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; + +public class DataApiInsertOneOp extends DataApiBaseOp { + private final Document doc; + private final String collectionName; + + public DataApiInsertOneOp(Database db, String collectionName, Document doc) { + super(db); + this.collectionName = collectionName; + this.doc = doc; + } + + @Override + public Object apply(long value) { + return db.getCollection(collectionName).insertOne(doc); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java new file mode 100644 index 000000000..dcb1966aa --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java @@ -0,0 +1,22 @@ +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; + +public class DataApiInsertOneVectorOp extends DataApiBaseOp { + private final Document doc; + private final String collectionName; + private float[] vector; + + public DataApiInsertOneVectorOp(Database db, String collectionName, Document doc, float[] vector) { + super(db); + this.collectionName = collectionName; + this.doc = doc; + this.vector = vector; + } + + @Override + public Object apply(long value) { + return db.getCollection(collectionName).insertOne(doc, vector); + } +} From 8fb7b80c265605434b9e363c9748dda6fa217272 Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Wed, 8 May 2024 18:45:02 -0400 Subject: [PATCH 06/19] refactor common code --- .../DataApiDeleteOneOpDispenser.java | 27 +++++++++---------- .../DataApiFindOneAndDeleteOpDispenser.java | 27 +++++++++---------- .../DataApiFindOneOpDispenser.java | 25 ++++++++--------- .../opdispensers/DataApiFindOpDispenser.java | 25 ++++++++--------- .../opdispensers/DataApiOpDispenser.java | 17 ++++++++++++ 5 files changed, 63 insertions(+), 58 deletions(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java index 3473267a8..f2de7714a 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java @@ -17,10 +17,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; -import com.datastax.astra.client.model.DeleteOneOptions; -import com.datastax.astra.client.model.Filter; -import com.datastax.astra.client.model.Filters; -import com.datastax.astra.client.model.Sorts; +import com.datastax.astra.client.model.*; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteOneOp; @@ -45,17 +42,8 @@ public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); - DeleteOneOptions options = new DeleteOneOptions(); - Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); - if (sortFunction.isPresent()) { - Map sortFields = sortFunction.get().apply(l); - String sortOrder = sortFields.get("type").toString(); - String sortField = sortFields.get("field").toString(); - switch(sortOrder) { - case "asc" -> options = options.sort(Sorts.ascending(sortField)); - case "desc" -> options = options.sort(Sorts.descending(sortField)); - } - } + DeleteOneOptions options = getDeleteOneOptions(op, l); + return new DataApiDeleteOneOp( db, db.getCollection(targetFunction.apply(l)), @@ -65,6 +53,15 @@ public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser { }; } + private DeleteOneOptions getDeleteOneOptions(ParsedOp op, long l) { + DeleteOneOptions options = new DeleteOneOptions(); + Sort sort = getSortFromOp(op, l); + if (sort != null) { + options = options.sort(sort); + } + return options; + } + @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java index 47e594ae4..b63a6386c 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java @@ -18,9 +18,8 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; -import com.datastax.astra.client.model.Filters; import com.datastax.astra.client.model.FindOneAndDeleteOptions; -import com.datastax.astra.client.model.Sorts; +import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneAndDeleteOp; @@ -28,8 +27,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser { @@ -45,17 +42,8 @@ public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); - FindOneAndDeleteOptions options = new FindOneAndDeleteOptions(); - Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); - if (sortFunction.isPresent()) { - Map sortFields = sortFunction.get().apply(l); - String sortOrder = sortFields.get("type").toString(); - String sortField = sortFields.get("field").toString(); - switch(sortOrder) { - case "asc" -> options = options.sort(Sorts.ascending(sortField)); - case "desc" -> options = options.sort(Sorts.descending(sortField)); - } - } + FindOneAndDeleteOptions options = getFindOneAndDeleteOptions(op, l); + return new DataApiFindOneAndDeleteOp( db, db.getCollection(targetFunction.apply(l)), @@ -65,6 +53,15 @@ public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser { }; } + private FindOneAndDeleteOptions getFindOneAndDeleteOptions(ParsedOp op, long l) { + FindOneAndDeleteOptions options = new FindOneAndDeleteOptions(); + Sort sort = getSortFromOp(op, l); + if (sort != null) { + options = options.sort(sort); + } + return options; + } + @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java index 87478e90b..a07c0be97 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -19,7 +19,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; import com.datastax.astra.client.model.FindOneOptions; -import com.datastax.astra.client.model.Sorts; +import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneOp; @@ -27,8 +27,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOneOpDispenser extends DataApiOpDispenser { @@ -43,17 +41,7 @@ public class DataApiFindOneOpDispenser extends DataApiOpDispenser { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); - FindOneOptions options = new FindOneOptions(); - Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); - if (sortFunction.isPresent()) { - Map sortFields = sortFunction.get().apply(l); - String sortOrder = sortFields.get("type").toString(); - String sortField = sortFields.get("field").toString(); - switch(sortOrder) { - case "asc" -> options = options.sort(Sorts.ascending(sortField)); - case "desc" -> options = options.sort(Sorts.descending(sortField)); - } - } + FindOneOptions options = getFindOneOptions(op, l); return new DataApiFindOneOp( db, db.getCollection(targetFunction.apply(l)), @@ -63,6 +51,15 @@ public class DataApiFindOneOpDispenser extends DataApiOpDispenser { }; } + private FindOneOptions getFindOneOptions(ParsedOp op, long l) { + FindOneOptions options = new FindOneOptions(); + Sort sort = getSortFromOp(op, l); + if (sort != null) { + options = options.sort(sort); + } + return options; + } + @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index adc2665f6..3c9af8fa3 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -19,7 +19,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; import com.datastax.astra.client.model.FindOptions; -import com.datastax.astra.client.model.Sorts; +import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiFindOp; @@ -27,8 +27,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class DataApiFindOpDispenser extends DataApiOpDispenser { @@ -43,17 +41,7 @@ public class DataApiFindOpDispenser extends DataApiOpDispenser { return (l) -> { Database db = spaceFunction.apply(l).getDatabase(); Filter filter = getFilterFromOp(op, l); - FindOptions options = new FindOptions(); - Optional> sortFunction = op.getAsOptionalFunction("sort", Map.class); - if (sortFunction.isPresent()) { - Map sortFields = sortFunction.get().apply(l); - String sortOrder = sortFields.get("type").toString(); - String sortField = sortFields.get("field").toString(); - switch(sortOrder) { - case "asc" -> options = options.sort(Sorts.ascending(sortField)); - case "desc" -> options = options.sort(Sorts.descending(sortField)); - } - } + FindOptions options = getFindOptions(op, l); return new DataApiFindOp( db, db.getCollection(targetFunction.apply(l)), @@ -63,6 +51,15 @@ public class DataApiFindOpDispenser extends DataApiOpDispenser { }; } + private FindOptions getFindOptions(ParsedOp op, long l) { + FindOptions options = new FindOptions(); + Sort sort = getSortFromOp(op, l); + if (sort != null) { + options = options.sort(sort); + } + return options; + } + @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index 7ab96ad52..c732b5833 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -18,6 +18,8 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.model.Filter; import com.datastax.astra.client.model.Filters; +import com.datastax.astra.client.model.Sort; +import com.datastax.astra.client.model.Sorts; import io.nosqlbench.adapter.dataapi.DataApiSpace; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; @@ -41,6 +43,21 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> sortFunction = op.getAsOptionalFunction("sort", Map.class); + if (sortFunction.isPresent()) { + Map sortFields = sortFunction.get().apply(l); + String sortOrder = sortFields.get("type").toString(); + String sortField = sortFields.get("field").toString(); + switch(sortOrder) { + case "asc" -> sort = Sorts.ascending(sortField); + case "desc" -> sort = Sorts.descending(sortField); + } + } + return sort; + } + protected Filter getFilterFromOp(ParsedOp op, long l) { Filter filter = null; Optional> filterFunction = op.getAsOptionalFunction("filters", Map.class); From 66ab79ecbe5b579fbc960d5e844015f74ccacdd7 Mon Sep 17 00:00:00 2001 From: MarkWolters Date: Wed, 8 May 2024 18:51:15 -0400 Subject: [PATCH 07/19] fixing rat check --- .../DataApiDeleteOneOpDispenser.java | 6 +++--- .../adapter/dataapi/ops/DataApiDeleteManyOp.java | 16 ++++++++++++++++ .../adapter/dataapi/ops/DataApiDeleteOneOp.java | 16 ++++++++++++++++ .../dataapi/ops/DataApiDropCollectionOp.java | 16 ++++++++++++++++ .../dataapi/ops/DataApiFindOneAndDeleteOp.java | 16 ++++++++++++++++ .../adapter/dataapi/ops/DataApiFindOneOp.java | 16 ++++++++++++++++ .../adapter/dataapi/ops/DataApiFindOp.java | 16 ++++++++++++++++ .../adapter/dataapi/ops/DataApiInsertOneOp.java | 16 ++++++++++++++++ .../dataapi/ops/DataApiInsertOneVectorOp.java | 16 ++++++++++++++++ 9 files changed, 131 insertions(+), 3 deletions(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java index f2de7714a..2cb71a3c2 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java @@ -17,7 +17,9 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; -import com.datastax.astra.client.model.*; +import com.datastax.astra.client.model.DeleteOneOptions; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteOneOp; @@ -25,8 +27,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Map; -import java.util.Optional; import java.util.function.LongFunction; public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java index cb67ccef5..deabceca8 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Collection; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java index 1e207d651..0e4c874fc 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Collection; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java index ba45f98a0..25caace45 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Database; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java index 5123527d1..e512d2865 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Collection; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java index 4ec93456d..17d1089d6 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Collection; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java index cfacd54ae..f3365edef 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Collection; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java index 13fc5893b..a1ab20ff4 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Database; diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java index dcb1966aa..4d21dcc68 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertOneVectorOp.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.nosqlbench.adapter.dataapi.ops; import com.datastax.astra.client.Database; From 464b6a8df39e801a093d49187e6c856efc0e8a4e Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Thu, 9 May 2024 10:55:59 -0400 Subject: [PATCH 08/19] find and insert ops --- .../DataApiFindOneAndUpdateOpDispenser.java | 26 +++++++++++- .../DataApiInsertManyOpDispenser.java | 32 +++++++++++++- .../opdispensers/DataApiOpDispenser.java | 35 ++++++++++++++-- .../ops/DataApiFindOneAndUpdateOp.java | 39 +++++++++++++++++ .../dataapi/ops/DataApiInsertManyOp.java | 42 +++++++++++++++++++ 5 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndUpdateOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertManyOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java index 7f5d6fb03..1b2fffccd 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndUpdateOpDispenser.java @@ -16,19 +16,43 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.Update; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneAndUpdateOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; public class DataApiFindOneAndUpdateOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindOneAndUpdateOpDispenser.class); + private final LongFunction opFunction; public DataApiFindOneAndUpdateOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + Filter filter = getFilterFromOp(op, l); + Update update = getUpdates(op, l); + + return new DataApiFindOneAndUpdateOp( + db, + db.getCollection(targetFunction.apply(l)), + filter, + update + ); + }; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java index 25b2a6537..e2aea7d51 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java @@ -16,19 +16,49 @@ package io.nosqlbench.adapter.dataapi.opdispensers; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.InsertManyOptions; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiInsertManyOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.List; import java.util.function.LongFunction; public class DataApiInsertManyOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiInsertManyOpDispenser.class); + private final LongFunction opFunction; + public DataApiInsertManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + List documents = new ArrayList<>(); + op.getAsRequiredFunction("documents", List.class).apply(l).forEach(o -> documents.add(Document.parse(o.toString()))); + return new DataApiInsertManyOp( + spaceFunction.apply(l).getDatabase(), + targetFunction.apply(l), + documents, + getInsertManyOptions(op, l) + ); + }; + } + + private InsertManyOptions getInsertManyOptions(ParsedOp op, long l) { + InsertManyOptions options = new InsertManyOptions(); + + return options; } @Override public DataApiBaseOp getOp(long value) { - return null; + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index c732b5833..f249824d1 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -16,10 +16,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; -import com.datastax.astra.client.model.Filter; -import com.datastax.astra.client.model.Filters; -import com.datastax.astra.client.model.Sort; -import com.datastax.astra.client.model.Sorts; +import com.datastax.astra.client.model.*; import io.nosqlbench.adapter.dataapi.DataApiSpace; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; @@ -93,4 +90,34 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> updatesFunction = op.getAsOptionalFunction("updates", Map.class); + if (updatesFunction.isPresent()) { + Map updates = updatesFunction.get().apply(l); + for (Map.Entry entry : updates.entrySet()) { + if (entry.getKey().equalsIgnoreCase("update")) { + Map updateFields = (Map) entry.getValue(); + switch (updateFields.get("operation").toString()) { + case "set" -> + update = Updates.set(updateFields.get("field").toString(), updateFields.get("value")); + case "inc" -> + update = Updates.inc(updateFields.get("field").toString(), (double) updateFields.get("value")); + case "unset" -> update = Updates.unset(updateFields.get("field").toString()); + case "addToSet" -> + update = Updates.addToSet(updateFields.get("field").toString(), updateFields.get("value")); + case "min" -> + update = Updates.min(updateFields.get("field").toString(), (double) updateFields.get("value")); + case "rename" -> + update = Updates.rename(updateFields.get("field").toString(), updateFields.get("value").toString()); + default -> logger.error("Operation " + updateFields.get("operation") + " not supported"); + } + } else { + logger.error("Filter " + entry.getKey() + " not supported"); + } + } + } + return update; + } + } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndUpdateOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndUpdateOp.java new file mode 100644 index 000000000..1ad55e00f --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndUpdateOp.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.*; + +public class DataApiFindOneAndUpdateOp extends DataApiBaseOp { + private final Collection collection; + private final Filter filter; + private final Update update; + + public DataApiFindOneAndUpdateOp(Database db, Collection collection, Filter filter, Update update) { + super(db); + this.collection = collection; + this.filter = filter; + this.update = update; + } + + @Override + public Object apply(long value) { + return collection.findOneAndUpdate(filter, update); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertManyOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertManyOp.java new file mode 100644 index 000000000..a3ceb4ef2 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiInsertManyOp.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.InsertManyOptions; + +import java.util.List; + +public class DataApiInsertManyOp extends DataApiBaseOp { + private final List documents; + private final String collectionName; + private final InsertManyOptions options; + + + public DataApiInsertManyOp(Database db, String collectionName, List documents, InsertManyOptions options) { + super(db); + this.collectionName = collectionName; + this.documents = documents; + this.options = options; + } + + @Override + public Object apply(long value) { + return db.getCollection(collectionName).insertMany(documents, options); + } +} From d9d728eadd87f72e747cbe9ebf2e5c49245ee69f Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Thu, 9 May 2024 17:42:29 -0400 Subject: [PATCH 09/19] implementing options behaviors --- .../DataApiDeleteOneOpDispenser.java | 8 ++++ .../DataApiFindOneAndDeleteOpDispenser.java | 5 +++ .../DataApiFindOneOpDispenser.java | 6 +++ .../opdispensers/DataApiFindOpDispenser.java | 6 +++ .../DataApiInsertManyOpDispenser.java | 17 ++++++++- .../opdispensers/DataApiOpDispenser.java | 38 +++++++++++++++++++ 6 files changed, 79 insertions(+), 1 deletion(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java index 2cb71a3c2..c5044a007 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java @@ -59,9 +59,17 @@ public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser { if (sort != null) { options = options.sort(sort); } + float[] vector = getVectorFromOp(op, l); + if (vector != null) { + options = options.vector(vector); + } return options; } + private float[] getVectorFromOp(ParsedOp op, long l) { + return getVectorValues(op.get("vector", l)); + } + @Override public DataApiBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java index b63a6386c..7a85f9995 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; import com.datastax.astra.client.model.FindOneAndDeleteOptions; +import com.datastax.astra.client.model.Projection; import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; @@ -59,6 +60,10 @@ public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser { if (sort != null) { options = options.sort(sort); } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java index a07c0be97..7731672bf 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; import com.datastax.astra.client.model.FindOneOptions; +import com.datastax.astra.client.model.Projection; import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; @@ -57,6 +58,11 @@ public class DataApiFindOneOpDispenser extends DataApiOpDispenser { if (sort != null) { options = options.sort(sort); } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + options.setIncludeSimilarity(true); return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index 3c9af8fa3..4c7bfd3c7 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapter.dataapi.opdispensers; import com.datastax.astra.client.Database; import com.datastax.astra.client.model.Filter; import com.datastax.astra.client.model.FindOptions; +import com.datastax.astra.client.model.Projection; import com.datastax.astra.client.model.Sort; import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; @@ -57,6 +58,11 @@ public class DataApiFindOpDispenser extends DataApiOpDispenser { if (sort != null) { options = options.sort(sort); } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + options.setIncludeSimilarity(true); return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java index e2aea7d51..10788cfea 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertManyOpDispenser.java @@ -27,6 +27,8 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; public class DataApiInsertManyOpDispenser extends DataApiOpDispenser { @@ -53,7 +55,20 @@ public class DataApiInsertManyOpDispenser extends DataApiOpDispenser { private InsertManyOptions getInsertManyOptions(ParsedOp op, long l) { InsertManyOptions options = new InsertManyOptions(); - + Optional> optionsFunction = op.getAsOptionalFunction("options", Map.class); + if (optionsFunction.isPresent()) { + Map optionFields = optionsFunction.get().apply(l); + for(Map.Entry entry: optionFields.entrySet()) { + switch(entry.getKey()) { + case "chunkSize"-> + options = options.chunkSize(Integer.parseInt(entry.getValue())); + case "concurrency" -> + options = options.concurrency(Integer.parseInt(entry.getValue())); + case "ordered" -> + options = options.ordered(Boolean.parseBoolean(entry.getValue())); + } + } + } return options; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index f249824d1..f13559a56 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -120,4 +120,42 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> projectionFunction = op.getAsOptionalFunction("projection", Map.class); + if (projectionFunction.isPresent()) { + Map> projectionFields = projectionFunction.get().apply(l); + for (Map.Entry> field : projectionFields.entrySet()) { + List includeFields = field.getValue(); + StringBuffer sb = new StringBuffer(); + for (String includeField : includeFields) { + sb.append(includeField).append(","); + } + sb.deleteCharAt(sb.length() - 1); + if (field.getKey().equalsIgnoreCase("include")) { + projection = Projections.include(sb.toString()); + } else if (field.getKey().equalsIgnoreCase("exclude")) { + projection = Projections.exclude(sb.toString()); + } else { + logger.error("Projection " + field + " not supported"); + } + } + } + return projection; + } + } From 455a938bc836635186c243b8c64f9a789a59bc6c Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 11:36:01 -0400 Subject: [PATCH 10/19] adding example yamls --- .../opdispensers/DataApiOpDispenser.java | 61 +++++++++++-------- .../activities/create_collection.yaml | 11 ++++ .../resources/activities/delete_many.yaml | 20 ++++++ .../main/resources/activities/delete_one.yaml | 0 .../resources/activities/drop_collection.yaml | 0 .../src/main/resources/activities/find.yaml | 0 .../main/resources/activities/find_one.yaml | 0 .../activities/find_one_and_delete.yaml | 0 .../activities/find_one_and_update.yaml | 0 .../resources/activities/insert_many.yaml | 0 .../main/resources/activities/insert_one.yaml | 0 .../activities/insert_one_vector.yaml | 0 12 files changed, 65 insertions(+), 27 deletions(-) create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_update.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index f13559a56..9ac931ced 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -57,35 +57,42 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> filterFunction = op.getAsOptionalFunction("filters", Map.class); + Optional> filterFunction = op.getAsOptionalFunction("filters", List.class); if (filterFunction.isPresent()) { - Map filters = filterFunction.get().apply(l); - List filterList = new ArrayList<>(); - String conjunction = "and"; - for (Map.Entry entry : filters.entrySet()) { - if (entry.getKey().equalsIgnoreCase("filter")) { - Map filterFields = (Map) entry.getValue(); - switch (filterFields.get("operation").toString()) { - case "lt" -> - filterList.add(Filters.lt(filters.get("field").toString(), (long) filters.get("value"))); - case "gt" -> - filterList.add(Filters.gt(filters.get("field").toString(), (long) filters.get("value"))); - case "eq" -> filterList.add(Filters.eq(filters.get("field").toString(), filters.get("value"))); - default -> logger.error("Operation not supported"); + List> filters = filterFunction.get().apply(l); + List andFilterList = new ArrayList<>(); + List orFilterList = new ArrayList<>(); + for (Map filterFields : filters) { + switch ((String)filterFields.get("conjunction")) { + case "and" -> { + switch (filterFields.get("operator").toString()) { + case "lt" -> + andFilterList.add(Filters.lt(filterFields.get("field").toString(), (long) filterFields.get("value"))); + case "gt" -> + andFilterList.add(Filters.gt(filterFields.get("field").toString(), (long) filterFields.get("value"))); + case "eq" -> + andFilterList.add(Filters.eq(filterFields.get("field").toString(), filterFields.get("value"))); + default -> logger.error(() -> "Operation " + filterFields.get("operator") + " not supported"); + } } - } else if (entry.getKey().equalsIgnoreCase("conjunction")) { - conjunction = (String) entry.getValue(); - } else { - logger.error("Filter " + entry.getKey() + " not supported"); + case "or" -> { + switch (filterFields.get("operator").toString()) { + case "lt" -> + orFilterList.add(Filters.lt(filterFields.get("field").toString(), (long) filterFields.get("value"))); + case "gt" -> + orFilterList.add(Filters.gt(filterFields.get("field").toString(), (long) filterFields.get("value"))); + case "eq" -> + orFilterList.add(Filters.eq(filterFields.get("field").toString(), filterFields.get("value"))); + default -> logger.error(() -> "Operation " + filterFields.get("operator") + " not supported"); + } + } + default -> logger.error(() -> "Conjunction " + filterFields.get("conjunction") + " not supported"); } } - if (conjunction.equalsIgnoreCase("and")) { - filter = Filters.and(filterList); - } else if (conjunction.equalsIgnoreCase("or")) { - filter = Filters.or(filterList); - } else { - logger.error("Conjunction " + conjunction + " not supported"); - } + if (!andFilterList.isEmpty()) + filter = Filters.and(andFilterList.toArray(new Filter[0])); + if (!orFilterList.isEmpty()) + filter = Filters.or(orFilterList.toArray(new Filter[0])); } return filter; } @@ -110,10 +117,10 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser update = Updates.rename(updateFields.get("field").toString(), updateFields.get("value").toString()); - default -> logger.error("Operation " + updateFields.get("operation") + " not supported"); + default -> logger.error(() -> "Operation " + updateFields.get("operation") + " not supported"); } } else { - logger.error("Filter " + entry.getKey() + " not supported"); + logger.error(() -> "Filter " + entry.getKey() + " not supported"); } } } diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml new file mode 100644 index 000000000..bee8b989a --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml @@ -0,0 +1,11 @@ +scenarios: + default: + create_collection: run driver=dataapi tags==blocks:create_collection cycles=1 + +blocks: + create_collection: + ops: + op1: + create_collection: "collectionName" + dimensions: 128 + similarity: "cosine" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml new file mode 100644 index 000000000..3149ea7d5 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml @@ -0,0 +1,20 @@ +scenarios: + default: + delete_many: run driver=dataapi tags==blocks:delete_many cycles=1 + +blocks: + delete_many: + ops: + op1: + delete_many: "collectionName" + filters: + - name: "filter1" + conjunction: "and" + operator: "lt" + field: "field1" + value: 10 + - name: "filter2" + conjunction: "or" + operator: "eq" + field: "field2" + value: "value2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_update.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_update.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml new file mode 100644 index 000000000..e69de29bb From 8b332fbed82477bd779f9fb4309a867bed3cd8f5 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 15:10:40 -0400 Subject: [PATCH 11/19] filling yamls --- .../dataapi/ops/DataApiDropCollectionOp.java | 4 +-- .../activities/create_collection.yaml | 2 +- .../resources/activities/delete_many.yaml | 6 ++--- .../main/resources/activities/delete_one.yaml | 13 ++++++++++ .../resources/activities/drop_collection.yaml | 9 +++++++ .../src/main/resources/activities/find.yaml | 25 +++++++++++++++++++ .../main/resources/activities/find_one.yaml | 25 +++++++++++++++++++ .../activities/find_one_and_delete.yaml | 25 +++++++++++++++++++ 8 files changed, 102 insertions(+), 7 deletions(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java index 25caace45..2c09c74af 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java @@ -20,9 +20,9 @@ import com.datastax.astra.client.Database; public class DataApiDropCollectionOp extends DataApiBaseOp { private final String collectionName; - public DataApiDropCollectionOp(Database db, String dbName) { + public DataApiDropCollectionOp(Database db, String collectionName) { super(db); - this.collectionName = dbName; + this.collectionName = collectionName; } @Override diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml index bee8b989a..23cc49592 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml @@ -7,5 +7,5 @@ blocks: ops: op1: create_collection: "collectionName" - dimensions: 128 + dimensions: 10 similarity: "cosine" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml index 3149ea7d5..18f981631 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_many.yaml @@ -8,13 +8,11 @@ blocks: op1: delete_many: "collectionName" filters: - - name: "filter1" - conjunction: "and" + - conjunction: "and" operator: "lt" field: "field1" value: 10 - - name: "filter2" - conjunction: "or" + - conjunction: "or" operator: "eq" field: "field2" value: "value2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml index e69de29bb..2b9b7a799 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/delete_one.yaml @@ -0,0 +1,13 @@ +scenarios: + default: + delete_one: run driver=dataapi tags==blocks:delete_one cycles=1 + +blocks: + delete_one: + ops: + op1: + delete_one: "collectionName" + sort: + type: "asc" + field: "field1" + vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml index e69de29bb..434bc92a7 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/drop_collection.yaml @@ -0,0 +1,9 @@ +scenarios: + default: + drop_collection: run driver=dataapi tags==blocks:drop_collection cycles=1 + +blocks: + drop_collection: + ops: + op1: + drop_collection: "collectionName" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml index e69de29bb..b98337a6f 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml @@ -0,0 +1,25 @@ +scenarios: + default: + find: run driver=dataapi tags==blocks:find cycles=10 + +blocks: + find: + ops: + op1: + find: "collectionName" + filters: + - conjunction: "and" + operator: "lt" + field: "field1" + value: 10 + - conjunction: "or" + operator: "eq" + field: "field2" + value: "value2" + sort: + type: "asc" + field: "field1" + projection: + include: + - "field1" + - "field2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml index e69de29bb..e1337fc81 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml @@ -0,0 +1,25 @@ +scenarios: + default: + find_one: run driver=dataapi tags==blocks:find_one cycles=10 + +blocks: + find_one: + ops: + op1: + find_one: "collectionName" + filters: + - conjunction: "and" + operator: "lt" + field: "field1" + value: 10 + - conjunction: "or" + operator: "eq" + field: "field2" + value: "value2" + sort: + type: "asc" + field: "field1" + projection: + include: + - "field1" + - "field2" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml index e69de29bb..e28b27c3c 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one_and_delete.yaml @@ -0,0 +1,25 @@ +scenarios: + default: + find_one_and_delete: run driver=dataapi tags==blocks:find_one_and_delete cycles=1 + +blocks: + find_one_and_delete: + ops: + op1: + find_one_and_delete: "collectionName" + filters: + - conjunction: "and" + operator: "lt" + field: "field1" + value: 10 + - conjunction: "or" + operator: "eq" + field: "field2" + value: "value2" + sort: + type: "asc" + field: "field1" + projection: + include: + - "field1" + - "field2" From 6799c196d1820195bca6b4d2ca7185c33660eeb7 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 15:48:48 -0400 Subject: [PATCH 12/19] more example yamls --- .../main/resources/activities/insert_many.yaml | 17 +++++++++++++++++ .../main/resources/activities/insert_one.yaml | 10 ++++++++++ .../resources/activities/insert_one_vector.yaml | 11 +++++++++++ 3 files changed, 38 insertions(+) diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml index e69de29bb..81a818d6f 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_many.yaml @@ -0,0 +1,17 @@ +scenarios: + default: + insert_many: run driver=dataapi tags==blocks:insert_many cycles=1 + +blocks: + insert_many: + ops: + op1: + insert_many: "collectionName" + options: + chunkSize: "1000" + concurrency: "10" + ordered: "false" + documents: + - "{valid json here}" + - "{valid json here}" + - "{valid json here}" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml index e69de29bb..567f06cd0 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one.yaml @@ -0,0 +1,10 @@ +scenarios: + default: + insert_one: run driver=dataapi tags==blocks:insert_one cycles=1 + +blocks: + insert_one: + ops: + op1: + insert_one: "collectionName" + document: "{valid json here}" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml index e69de29bb..a9bce8200 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/insert_one_vector.yaml @@ -0,0 +1,11 @@ +scenarios: + default: + insert_one: run driver=dataapi tags==blocks:insert_one cycles=1 + +blocks: + insert_one: + ops: + op1: + insert_one: "collectionName" + document: "{valid json here}" + vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" From d4462bb8f37743fe5c21a39ce113779eaf046a67 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 10 May 2024 16:39:06 -0500 Subject: [PATCH 13/19] incremental update --- .run/astra DAPI rampup.run.xml | 31 ++++ .../adapter/dataapi/DataApiOpMapper.java | 2 +- .../adapter/dataapi/DataApiSpace.java | 42 +++++- .../dataapi/ops/DataApiDropCollectionOp.java | 3 + .../resources/activities/astra_kv_dapi.yaml | 142 ++++++++++++++++++ nb-adapters/nb-adapters-included/pom.xml | 14 ++ nb-adapters/pom.xml | 2 +- .../api/config/standard/NBConfiguration.java | 2 +- .../nb/api/config/standard/Param.java | 3 + 9 files changed, 233 insertions(+), 8 deletions(-) create mode 100644 .run/astra DAPI rampup.run.xml create mode 100644 nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml diff --git a/.run/astra DAPI rampup.run.xml b/.run/astra DAPI rampup.run.xml new file mode 100644 index 000000000..8bd74b03d --- /dev/null +++ b/.run/astra DAPI rampup.run.xml @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index 7e1ba4e1a..d212df109 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -40,7 +40,7 @@ public class DataApiOpMapper implements OpMapper { DataApiOpType.class, String.class, "type", - "target" + "collection" ); logger.debug(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); return switch (typeAndTarget.enumId) { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java index ea0c5e1e9..e1cd61d30 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java @@ -22,6 +22,7 @@ import io.nosqlbench.nb.api.config.standard.ConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfigModel; import io.nosqlbench.nb.api.config.standard.NBConfiguration; import io.nosqlbench.nb.api.config.standard.Param; +import io.nosqlbench.nb.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +41,7 @@ public class DataApiSpace { private DataAPIClient dataAPIClient; private Database database; private String namespace; + public DataApiSpace(String name, NBConfiguration cfg) { this.config = cfg; this.name = name; @@ -67,7 +69,26 @@ public class DataApiSpace { } private void setApiEndpoint() { - this.astraApiEndpoint = config.get("astraApiEndpoint"); + Optional epConfig = config.getOptional("astraApiEndpoint"); + Optional epFileConfig = config.getOptional("astraApiEndpointFile"); + if (epConfig.isPresent() && epFileConfig.isPresent()) { + throw new BasicError("You can only configure one of astraApiEndpoint or astraApiEndpointFile"); + } + if (epConfig.isEmpty() && epFileConfig.isEmpty()) { + throw new BasicError("You must configure one of astraApiEndpoint or astraApiEndpointFile"); + } + epFileConfig + .map(Path::of) + .map(p -> { + try { + return Files.readString(p); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .map(String::trim) + .ifPresent(ep -> this.astraApiEndpoint = ep); + epConfig.ifPresent(ep -> this.astraApiEndpoint = ep); } private void setNamespace() { @@ -78,7 +99,7 @@ public class DataApiSpace { private void setToken() { String tokenFileContents = null; Optional tokenFilePath = config.getOptional("astraTokenFile"); - if(tokenFilePath.isPresent()) { + if (tokenFilePath.isPresent()) { Path path = Paths.get(tokenFilePath.get()); try { tokenFileContents = Files.readAllLines(path).getFirst(); @@ -94,16 +115,27 @@ public class DataApiSpace { public static NBConfigModel getConfigModel() { return ConfigModel.of(DataApiSpace.class) .add( - Param.optional("astraTokenFile", String.class, "file to load the Astra token from") + Param.optional("astraTokenFile", String.class) + .setDescription("file to load the Astra token from") ) .add( - Param.optional("astraToken",String.class) + Param.optional("astraToken", String.class) .setDescription("the Astra token used to connect to the database") ) .add( - Param.defaultTo("astraApiEndpoint", String.class) + Param.optional("astraApiEndpoint", String.class) .setDescription("the API endpoint for the Astra database") ) + .add( + Param.optional("astraApiEndpointFile", String.class) + .setDescription("file to load the API endpoint for the Astra database") + ) + + .add( + Param.defaultTo("namespace", "default_namespace") + .setDescription("The Astra namespace to use") + + ) .asReadOnly(); } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java index 2c09c74af..effd4236b 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDropCollectionOp.java @@ -28,6 +28,9 @@ public class DataApiDropCollectionOp extends DataApiBaseOp { @Override public Object apply(long value) { Boolean exists = db.collectionExists(collectionName); + // TODO: we need to remove these from the ops when we can, because this hides additional ops which + // should be surfaced in the test definition. Condition operations should be provided with clear views + // at the workload template level if (exists) { db.dropCollection(collectionName); } diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml new file mode 100644 index 000000000..15f881a8b --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml @@ -0,0 +1,142 @@ +min_version: "5.21.0" + +description: | + A basic workload that uses the DataStax Data API Client in Java, emulating what + applications would do in the native stack. + TEMPLATE(collection,keyvalue) + +scenarios: + default: + schema: run driver=dataapi tags==block:schema threads==1 cycles==UNDEF + rampup: run driver=dataapi tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto +# main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto + +# kv_dapi: +# kv_dapi_schema: run driver=http tags==block:schema threads==1 cycles==UNDEF +# kv_dapi_rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto +# kv_dapi_main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto + +# basic_check: +# schema: run driver=http tags==block:schema threads==1 cycles==UNDEF +# rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10) threads=auto +# main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,10) threads=auto + +bindings: + # To enable an optional weighted set of hosts in place of a load balancer + # Examples + # single host: jsonapi_host=host1 + # multiple hosts: jsonapi_host=host1,host2,host3 + # multiple weighted hosts: jsonapi_host=host1:3,host2:7 + weighted_hosts: WeightedStrings('<>>>') + + seq_key: Mod(TEMPLATE(keycount,50000000000L)); ToString() -> String + seq_value: Hash(); Mod(TEMPLATE(valuecount,50000000000L)); ToString() -> String + rw_key: TEMPLATE(keydist,Uniform(0,50000000000L)); ToString() -> String + rw_value: Hash(); TEMPLATE(valdist,Uniform(0,50000000000L)); ToString() -> String + vector_value: HashedFloatVectors(<>); ToCqlVector() + + request_id: ToHashedUUID(); ToString(); + token: Discard(); Token('TEMPLATE(auth_token)','TEMPLATE(uri,http://localhost:8081/v1/auth)', 'TEMPLATE(uid,cassandra)', 'TEMPLATE(pswd,cassandra)'); + + params: + cl: TEMPLATE(cl,LOCAL_QUORUM) + +blocks: + reset_schema: + ops: + drop_index: + raw: |- + DROP INDEX IF EXISTS TEMPLATE(keyspace, baselines).TEMPLATE(table,keyvalue)_value_idx; + drop-table: + raw: |- + DROP TABLE IF EXISTS TEMPLATE(keyspace, baselines).TEMPLATE(table,keyvalue); + + schema: + ops: + delete_collection_op: + delete_collection: "TEMPLATE(collection)" + create_collection_op: + create_collection: "TEMPLATE(collection)" + + rampup: + ops: + insert_one_op: + insert_one: "TEMPLATE(collection)" + document: + _id: "{seq_key}" + value: "{seq_value}" + rampup-uuid: + ops: + insert_one_op: + insert-one: "TEMPLATE(collection)" + document: + value: "{seq_value}" + main_read: + params: + ratio: 5 + ops: + find_op: + find: "TEMPLATE(collection)" + filter: + _id: "{rw_key}" + schema_with_vector: + ops: + delete_collection_op: + delete_collection: "TEMPLATE(collection)" + create_collection_op: + create_collection: "TEMPLATE(collection)" + options: + vector: + size: 1536 + rampup_with_vector: + ops: + insert_one_op: + document: + _id: "{seq_key}" + value: "{seq_value}" + $vector: "{vector_value}" + + rampup_with_vector_uuid: + ops: + insert_one_op: + insert_one: "TEMPLATE(collection)" + document: + value: "{seq_value}" + $vector: "{vector_value}" + main_read_with_vector: + ops: + find_op: + find: "TEMPLATE(collection)" + filter: + _id: "{rw_key}" + main_ann_with_vector_limit_20: + params: + ratio: 5 + ops: + find_op: + find: "TEMPLATE(collection)" + sort: + $vector: "{vector_value}" + options: + limit: 20 + schema_with_text_sai: + ops: + delete_collection_op: + delete_collection: "TEMPLATE(collection)" + create_collection_op: + create_collection: "TEMPLATE(collection)" + rampup_with_text_sai: + ops: + insert_one_op: + insert_one: "TEMPLATE(collection)" + document: + _id: "{seq_key}" + value: "{seq_value}" + main_read_with_text_sai: + params: + ratio: 5 + ops: + find_op: + find: "TEMPLATE(collection)" + filter: + value: "{rw_value}" diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 3721199c0..e1fdd0093 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -141,6 +141,20 @@ + + adapter-dataapi-include + + true + + + + io.nosqlbench + adapter-dataapi + ${revision} + + + + adapter-dynamodb-include diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index 97a84dfb5..f2c0d27fc 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -175,7 +175,7 @@ - adapter-dataapi + adapter-dataapi-module true diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java index 8e4ef25c4..a5da84bc6 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/NBConfiguration.java @@ -178,7 +178,7 @@ public class NBConfiguration { } } } else { - throw new NBConfigError("Parameter was not found for " + Arrays.toString(names) + "."); + throw new NBConfigError("Parameter definition was not found for " + Arrays.toString(names) + "."); } } if (o == null) { diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/Param.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/Param.java index 8a9b4fc37..0da1dcc19 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/Param.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/config/standard/Param.java @@ -136,6 +136,9 @@ public class Param { * @return */ public static Param defaultTo(String name, V defaultValue) { + if (defaultValue instanceof Class clazz) { + throw new RuntimeException("This is not a supported type."); + } return new Param(List.of(name), (Class) defaultValue.getClass(), null, true, defaultValue, null); } From d5a96e81c409d351796b8fc926ae31bf952af7d6 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 17:49:20 -0400 Subject: [PATCH 14/19] vectorizing find one op --- .../opdispensers/DataApiFindOneOpDispenser.java | 3 ++- .../opdispensers/DataApiOpDispenser.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java index 7731672bf..e79cefc82 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -55,8 +55,9 @@ public class DataApiFindOneOpDispenser extends DataApiOpDispenser { private FindOneOptions getFindOneOptions(ParsedOp op, long l) { FindOneOptions options = new FindOneOptions(); Sort sort = getSortFromOp(op, l); + float[] vector = getVectorValues(op, l); if (sort != null) { - options = options.sort(sort); + options = vector != null ? options.sort(vector, sort) : options.sort(sort); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index 9ac931ced..90a32d9df 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -127,6 +127,11 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser vectorValuesList = (List) rawVectorValues; + vectorValues = new float[vectorValuesList.size()]; + for (int i = 0; i < vectorValuesList.size(); i++) { + vectorValues[i] = Float.parseFloat(vectorValuesList.get(i).toString()); + } + return vectorValues; + } + protected Projection[] getProjectionFromOp(ParsedOp op, long l) { Projection[] projection = null; Optional> projectionFunction = op.getAsOptionalFunction("projection", Map.class); From e6d2c35f5a38c73ff32c96a5766b3b82721da215 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 18:00:13 -0400 Subject: [PATCH 15/19] fixed condition when find options consist of vector only --- .../dataapi/opdispensers/DataApiFindOneOpDispenser.java | 2 ++ .../adapter/dataapi/opdispensers/DataApiFindOpDispenser.java | 5 ++++- .../adapter-dataapi/src/main/resources/activities/find.yaml | 1 + .../src/main/resources/activities/find_one.yaml | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java index e79cefc82..687a9b91a 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java @@ -58,6 +58,8 @@ public class DataApiFindOneOpDispenser extends DataApiOpDispenser { float[] vector = getVectorValues(op, l); if (sort != null) { options = vector != null ? options.sort(vector, sort) : options.sort(sort); + } else if (vector != null) { + options = options.sort(vector); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index 4c7bfd3c7..99dcb95ad 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -55,8 +55,11 @@ public class DataApiFindOpDispenser extends DataApiOpDispenser { private FindOptions getFindOptions(ParsedOp op, long l) { FindOptions options = new FindOptions(); Sort sort = getSortFromOp(op, l); + float[] vector = getVectorValues(op, l); if (sort != null) { - options = options.sort(sort); + options = vector != null ? options.sort(vector, sort) : options.sort(sort); + } else if (vector != null) { + options = options.sort(vector); } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml index b98337a6f..72b406dcd 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/find.yaml @@ -19,6 +19,7 @@ blocks: sort: type: "asc" field: "field1" + vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" projection: include: - "field1" diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml index e1337fc81..e86bb16a3 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/find_one.yaml @@ -19,6 +19,7 @@ blocks: sort: type: "asc" field: "field1" + vector: "1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0" projection: include: - "field1" From 101e9d93facc5f33ef091e2dda8750b8455ea3c1 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 18:22:01 -0400 Subject: [PATCH 16/19] adding vector find ops --- .../DataApiFindVectorFilterOpDispenser.java | 82 +++++++++++++++++++ .../adapter/dataapi/DataApiOpMapper.java | 3 + .../DataApiFindVectorOpDispenser.java | 79 ++++++++++++++++++ .../ops/DataApiFindVectorFilterOp.java | 42 ++++++++++ .../dataapi/ops/DataApiFindVectorOp.java | 41 ++++++++++ .../adapter/dataapi/ops/DataApiOpType.java | 3 + 6 files changed, 250 insertions(+) create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiFindVectorFilterOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorFilterOp.java create mode 100644 nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorOp.java diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiFindVectorFilterOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiFindVectorFilterOpDispenser.java new file mode 100644 index 000000000..e6cfe4b5d --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiFindVectorFilterOpDispenser.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOptions; +import com.datastax.astra.client.model.Projection; +import com.datastax.astra.client.model.Sort; +import io.nosqlbench.adapter.dataapi.opdispensers.DataApiOpDispenser; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindVectorFilterOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiFindVectorFilterOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindVectorFilterOpDispenser.class); + private final LongFunction opFunction; + public DataApiFindVectorFilterOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + float[] vector = getVectorValues(op, l); + Filter filter = getFilterFromOp(op, l); + int limit = getLimit(op, l); + return new DataApiFindVectorFilterOp( + db, + db.getCollection(targetFunction.apply(l)), + vector, + limit, + filter + ); + }; + } + + private int getLimit(ParsedOp op, long l) { + return op.getConfigOr("limit", 100, l); + } + + private FindOptions getFindOptions(ParsedOp op, long l) { + FindOptions options = new FindOptions(); + Sort sort = getSortFromOp(op, l); + float[] vector = getVectorValues(op, l); + if (sort != null) { + options = vector != null ? options.sort(vector, sort) : options.sort(sort); + } else if (vector != null) { + options = options.sort(vector); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + options.setIncludeSimilarity(true); + return options; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java index d212df109..190b6f79f 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiOpMapper.java @@ -47,10 +47,13 @@ public class DataApiOpMapper implements OpMapper { case create_collection -> new DataApiCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); case insert_many -> new DataApiInsertManyOpDispenser(adapter, op, typeAndTarget.targetFunction); case insert_one -> new DataApiInsertOneOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert_one_vector -> new DataApiInsertOneVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); case find -> new DataApiFindOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_one -> new DataApiFindOneOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_one_and_delete -> new DataApiFindOneAndDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction); case find_one_and_update -> new DataApiFindOneAndUpdateOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_vector -> new DataApiFindVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); + case find_vector_filter -> new DataApiFindVectorFilterOpDispenser(adapter, op, typeAndTarget.targetFunction); case update_one -> new DataApiUpdateOneOpDispenser(adapter, op, typeAndTarget.targetFunction); case update_many -> new DataApiUpdateManyOpDispenser(adapter, op, typeAndTarget.targetFunction); case delete_one -> new DataApiDeleteOneOpDispenser(adapter, op, typeAndTarget.targetFunction); diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java new file mode 100644 index 000000000..5aa65f912 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindVectorOpDispenser.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.opdispensers; + +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.FindOptions; +import com.datastax.astra.client.model.Projection; +import com.datastax.astra.client.model.Sort; +import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter; +import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp; +import io.nosqlbench.adapter.dataapi.ops.DataApiFindVectorOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class DataApiFindVectorOpDispenser extends DataApiOpDispenser { + private static final Logger logger = LogManager.getLogger(DataApiFindVectorOpDispenser.class); + private final LongFunction opFunction; + public DataApiFindVectorOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> { + Database db = spaceFunction.apply(l).getDatabase(); + float[] vector = getVectorValues(op, l); + int limit = getLimit(op, l); + return new DataApiFindVectorOp( + db, + db.getCollection(targetFunction.apply(l)), + vector, + limit + ); + }; + } + + private int getLimit(ParsedOp op, long l) { + return op.getConfigOr("limit", 100, l); + } + + private FindOptions getFindOptions(ParsedOp op, long l) { + FindOptions options = new FindOptions(); + Sort sort = getSortFromOp(op, l); + float[] vector = getVectorValues(op, l); + if (sort != null) { + options = vector != null ? options.sort(vector, sort) : options.sort(sort); + } else if (vector != null) { + options = options.sort(vector); + } + Projection[] projection = getProjectionFromOp(op, l); + if (projection != null) { + options = options.projection(projection); + } + options.setIncludeSimilarity(true); + return options; + } + + @Override + public DataApiBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorFilterOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorFilterOp.java new file mode 100644 index 000000000..5f7c21bd6 --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorFilterOp.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; + +public class DataApiFindVectorFilterOp extends DataApiBaseOp { + private final Collection collection; + private final float[] vector; + private final int limit; + private final Filter filter; + + public DataApiFindVectorFilterOp(Database db, Collection collection, float[] vector, int limit, Filter filter) { + super(db); + this.collection = collection; + this.vector = vector; + this.limit = limit; + this.filter = filter; + } + + @Override + public Object apply(long value) { + return collection.find(filter, vector, limit); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorOp.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorOp.java new file mode 100644 index 000000000..7c08a35fe --- /dev/null +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindVectorOp.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.dataapi.ops; + +import com.datastax.astra.client.Collection; +import com.datastax.astra.client.Database; +import com.datastax.astra.client.model.Document; +import com.datastax.astra.client.model.Filter; +import com.datastax.astra.client.model.FindOptions; + +public class DataApiFindVectorOp extends DataApiBaseOp { + private final Collection collection; + private final float[] vector; + private final int limit; + + public DataApiFindVectorOp(Database db, Collection collection, float[] vector, int limit) { + super(db); + this.collection = collection; + this.vector = vector; + this.limit = limit; + } + + @Override + public Object apply(long value) { + return collection.find(vector, limit); + } +} diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java index c2f40fe2a..4ddae3b83 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiOpType.java @@ -20,10 +20,13 @@ public enum DataApiOpType { create_collection, insert_many, insert_one, + insert_one_vector, find, find_one, find_one_and_delete, find_one_and_update, + find_vector, + find_vector_filter, update_one, update_many, delete_one, From 031fd5fc5908bdf3df657c6d7d609066ee869fd5 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 10 May 2024 18:48:44 -0400 Subject: [PATCH 17/19] bug fix for similarity function --- .../opdispensers/DataApiCreateCollectionOpDispenser.java | 2 +- .../src/main/resources/activities/create_collection.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java index d20a5521b..14e30e5f1 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java @@ -51,7 +51,7 @@ public class DataApiCreateCollectionOpDispenser extends DataApiOpDispenser { Optional> simFunc = op.getAsOptionalFunction("similarity", String.class); if (simFunc.isPresent()) { LongFunction sf = simFunc.get(); - optionsBldr.vectorSimilarity(SimilarityMetric.valueOf(sf.apply(l))); + optionsBldr.vectorSimilarity(SimilarityMetric.fromValue(sf.apply(l))); } DataApiCreateCollectionOp dataApiCreateCollectionOp = diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml index 23cc49592..66ebc5fdf 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/create_collection.yaml @@ -8,4 +8,4 @@ blocks: op1: create_collection: "collectionName" dimensions: 10 - similarity: "cosine" + similarity: "COSINE" From 0edc60e9d08aacb3f1d3fd3b5d40ef4d694f81ac Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 10 May 2024 19:16:03 -0500 Subject: [PATCH 18/19] basic schema(+vectors), rampup (+vectors), and read support --- ...ml => astra DAPI rampup threads=1.run.xml} | 20 +- .run/astra DAPI read.run.xml | 15 ++ ...a DAPI schema collection=baselines.run.xml | 15 ++ .../opdispensers/DataApiFindOpDispenser.java | 12 +- .../DataApiInsertOneOpDispenser.java | 6 +- .../DataApiInsertOneVectorOpDispenser.java | 10 +- .../opdispensers/DataApiOpDispenser.java | 7 +- .../adapter/dataapi/ops/DataApiFindOp.java | 15 +- .../resources/activities/astra_kv_dapi.yaml | 173 ++++++++++-------- .../api/templating/ParsedTemplateMapTest.java | 16 ++ 10 files changed, 179 insertions(+), 110 deletions(-) rename .run/{astra DAPI rampup.run.xml => astra DAPI rampup threads=1.run.xml} (50%) create mode 100644 .run/astra DAPI read.run.xml create mode 100644 .run/astra DAPI schema collection=baselines.run.xml diff --git a/.run/astra DAPI rampup.run.xml b/.run/astra DAPI rampup threads=1.run.xml similarity index 50% rename from .run/astra DAPI rampup.run.xml rename to .run/astra DAPI rampup threads=1.run.xml index 8bd74b03d..e13f0c550 100644 --- a/.run/astra DAPI rampup.run.xml +++ b/.run/astra DAPI rampup threads=1.run.xml @@ -1,21 +1,5 @@ - - - + - + \ No newline at end of file diff --git a/.run/astra DAPI read.run.xml b/.run/astra DAPI read.run.xml new file mode 100644 index 000000000..ae12e2961 --- /dev/null +++ b/.run/astra DAPI read.run.xml @@ -0,0 +1,15 @@ + + + + + + \ No newline at end of file diff --git a/.run/astra DAPI schema collection=baselines.run.xml b/.run/astra DAPI schema collection=baselines.run.xml new file mode 100644 index 000000000..e50fb39a0 --- /dev/null +++ b/.run/astra DAPI schema collection=baselines.run.xml @@ -0,0 +1,15 @@ + + + + + + \ No newline at end of file diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java index 99dcb95ad..d615cc656 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java @@ -55,11 +55,13 @@ public class DataApiFindOpDispenser extends DataApiOpDispenser { private FindOptions getFindOptions(ParsedOp op, long l) { FindOptions options = new FindOptions(); Sort sort = getSortFromOp(op, l); - float[] vector = getVectorValues(op, l); - if (sort != null) { - options = vector != null ? options.sort(vector, sort) : options.sort(sort); - } else if (vector != null) { - options = options.sort(vector); + if (op.isDefined("vector")) { + float[] vector = getVectorValues(op, l); + if (sort != null) { + options = vector != null ? options.sort(vector, sort) : options.sort(sort); + } else if (vector != null) { + options = options.sort(vector); + } } Projection[] projection = getProjectionFromOp(op, l); if (projection != null) { diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java index 2c43ca8ee..3c24357c2 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneOpDispenser.java @@ -24,6 +24,7 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Map; import java.util.function.LongFunction; public class DataApiInsertOneOpDispenser extends DataApiOpDispenser { @@ -36,12 +37,13 @@ public class DataApiInsertOneOpDispenser extends DataApiOpDispenser { } private LongFunction createOpFunction(ParsedOp op) { + LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); return (l) -> { - Document.parse(op.get("document", l)); return new DataApiInsertOneOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), - Document.parse(op.get("document", l)) + docFunc.apply(l) ); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java index 040dcbc01..a735bcf87 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiInsertOneVectorOpDispenser.java @@ -25,6 +25,7 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Map; import java.util.function.LongFunction; public class DataApiInsertOneVectorOpDispenser extends DataApiOpDispenser { @@ -37,14 +38,15 @@ public class DataApiInsertOneVectorOpDispenser extends DataApiOpDispenser { } private LongFunction createOpFunction(ParsedOp op) { + LongFunction docMapFunc = op.getAsRequiredFunction("document", Map.class); + LongFunction docFunc = (long m) -> new Document(docMapFunc.apply(m)); + LongFunction vectorF= op.getAsRequiredFunction("vector", float[].class); return (l) -> { - Document.parse(op.get("document", l)); - float[] vector = op.get("vector", l); return new DataApiInsertOneVectorOp( spaceFunction.apply(l).getDatabase(), targetFunction.apply(l), - Document.parse(op.get("document", l)), - vector + docFunc.apply(l), + vectorF.apply(l) ); }; } diff --git a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java index 90a32d9df..23f43fe43 100644 --- a/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java +++ b/nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java @@ -56,8 +56,11 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser> filterFunction = op.getAsOptionalFunction("filters", List.class); + Optional> filterFunction = op.getAsOptionalFunction("filters", List.class).or( + () -> op.getAsOptionalFunction("filter",List.class) + ); if (filterFunction.isPresent()) { List> filters = filterFunction.get().apply(l); List andFilterList = new ArrayList<>(); @@ -143,7 +146,7 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser collection; @@ -36,7 +36,10 @@ public class DataApiFindOp extends DataApiBaseOp { } @Override - public Object apply(long value) { - return collection.find(filter, options); + public List apply(long value) { + // TODO: make sure we're traversing result data + FindIterable documentStream = collection.find(filter, options); + List documents = StreamSupport.stream(documentStream.spliterator(), false).toList(); + return documents; } } diff --git a/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml b/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml index 15f881a8b..3572b0e39 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml +++ b/nb-adapters/adapter-dataapi/src/main/resources/activities/astra_kv_dapi.yaml @@ -3,13 +3,19 @@ min_version: "5.21.0" description: | A basic workload that uses the DataStax Data API Client in Java, emulating what applications would do in the native stack. + TEMPLATE(cardinality,1000) TEMPLATE(collection,keyvalue) + TEMPLATE(dimensions,1536) + TEMPLATE(similarity,COSINE) + TEMPLATE(keycount,TEMPLATE(cardinality)) + TEMPLATE(valuecount,TEMPLATE(cardinality)) scenarios: default: schema: run driver=dataapi tags==block:schema threads==1 cycles==UNDEF - rampup: run driver=dataapi tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto -# main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto + rampup: run driver=dataapi tags==block:rampup cycles===TEMPLATE(rampup-cycles,1000) threads=auto + rampup_vector: run driver=dataapi tags==block:rampup_vector cycles===TEMPLATE(rampup-cycles,TEMPLATE(cardinality)) threads=auto + find_key: run driver=dataapi tags==block:find_key cycles===TEMPLATE(main-cycles,1000) threads=auto # kv_dapi: # kv_dapi_schema: run driver=http tags==block:schema threads==1 cycles==UNDEF @@ -27,13 +33,16 @@ bindings: # single host: jsonapi_host=host1 # multiple hosts: jsonapi_host=host1,host2,host3 # multiple weighted hosts: jsonapi_host=host1:3,host2:7 - weighted_hosts: WeightedStrings('<>>>') + weighted_hosts: WeightedStrings('TEMPLATE(jsonapi_host,TEMPLATE(stargate_host,localhost))') - seq_key: Mod(TEMPLATE(keycount,50000000000L)); ToString() -> String - seq_value: Hash(); Mod(TEMPLATE(valuecount,50000000000L)); ToString() -> String - rw_key: TEMPLATE(keydist,Uniform(0,50000000000L)); ToString() -> String - rw_value: Hash(); TEMPLATE(valdist,Uniform(0,50000000000L)); ToString() -> String - vector_value: HashedFloatVectors(<>); ToCqlVector() + seq_key: Mod(TEMPLATE(keycount); ToString() -> String +# seq_key: Mod(TEMPLATE(keycount,50000000000L)); + seq_value: Hash(); Mod(TEMPLATE(valuecount); ToString() -> String +# rw_key: TEMPLATE(keydist,Uniform(0,50000000000L)); + rw_key: TEMPLATE(keydist,Uniform(0,TEMPLATE(keycount))); ToString() -> String + rw_key_num: TEMPLATE(keydist,Uniform(0,TEMPLATE(keycount))); + rw_value: Hash(); TEMPLATE(valdist,Uniform(0,TEMPLATE(valuecount)); ToString() -> String + vector_value: HashedFloatVectors(TEMPLATE(dimensions,1536)); request_id: ToHashedUUID(); ToString(); token: Discard(); Token('TEMPLATE(auth_token)','TEMPLATE(uri,http://localhost:8081/v1/auth)', 'TEMPLATE(uid,cassandra)', 'TEMPLATE(pswd,cassandra)'); @@ -57,6 +66,18 @@ blocks: delete_collection: "TEMPLATE(collection)" create_collection_op: create_collection: "TEMPLATE(collection)" + dimensions: TEMPLATE(dimensions) + similarity: TEMPLATE(similarity) + +# separate these cases later, when you can recreate the same collection name with/without vector support +# schema_vector: +# ops: +# delete_collection_op: +# delete_collection: "TEMPLATE(collection)" +# create_collection_op: +# create_collection: "TEMPLATE(collection)" +# dimensions: TEMPLATE(dimensions) +# similarity: TEMPLATE(similarity) rampup: ops: @@ -65,78 +86,84 @@ blocks: document: _id: "{seq_key}" value: "{seq_value}" - rampup-uuid: - ops: - insert_one_op: - insert-one: "TEMPLATE(collection)" - document: - value: "{seq_value}" - main_read: +# rampup-uuid: +# ops: +# insert_one_op: +# insert-one: "TEMPLATE(collection)" +# document: +# value: "{seq_value}" + find_key: params: ratio: 5 ops: find_op: find: "TEMPLATE(collection)" - filter: - _id: "{rw_key}" - schema_with_vector: - ops: - delete_collection_op: - delete_collection: "TEMPLATE(collection)" - create_collection_op: - create_collection: "TEMPLATE(collection)" - options: - vector: - size: 1536 - rampup_with_vector: + filters: + - conjunction: "and" + operator: "lt" + field: "_id" + value: "{rw_key_num}" +# schema_with_vector: +# ops: +# delete_collection_op: +# delete_collection: "TEMPLATE(collection)" +# create_collection_op: +# create_collection: "TEMPLATE(collection)" +# options: +# vector: +# size: 1536 + rampup_vector: ops: insert_one_op: + insert_one_vector: "TEMPLATE(collection)" + vector: "{vector_value}" document: _id: "{seq_key}" value: "{seq_value}" - $vector: "{vector_value}" - - rampup_with_vector_uuid: - ops: - insert_one_op: - insert_one: "TEMPLATE(collection)" - document: - value: "{seq_value}" - $vector: "{vector_value}" - main_read_with_vector: - ops: - find_op: - find: "TEMPLATE(collection)" - filter: - _id: "{rw_key}" - main_ann_with_vector_limit_20: - params: - ratio: 5 - ops: - find_op: - find: "TEMPLATE(collection)" - sort: - $vector: "{vector_value}" - options: - limit: 20 - schema_with_text_sai: - ops: - delete_collection_op: - delete_collection: "TEMPLATE(collection)" - create_collection_op: - create_collection: "TEMPLATE(collection)" - rampup_with_text_sai: - ops: - insert_one_op: - insert_one: "TEMPLATE(collection)" - document: - _id: "{seq_key}" - value: "{seq_value}" - main_read_with_text_sai: - params: - ratio: 5 - ops: - find_op: - find: "TEMPLATE(collection)" - filter: - value: "{rw_value}" +# +# rampup_with_vector_uuid: +# ops: +# insert_one_op: +# insert_one: "TEMPLATE(collection)" +# document: +# value: "{seq_value}" +# vector: "{vector_value}" +# +# main_read_with_vector: +# ops: +# find_op: +# find: "TEMPLATE(collection)" +# filter: +# _id: "{rw_key}" +# +# main_ann_with_vector_limit_20: +# params: +# ratio: 5 +# ops: +# find_op: +# find: "TEMPLATE(collection)" +# sort: +# vector: "{vector_value}" +# options: +# limit: 20 +# schema_with_text_sai: +# ops: +# delete_collection_op: +# delete_collection: "TEMPLATE(collection)" +# create_collection_op: +# create_collection: "TEMPLATE(collection)" +# rampup_with_text_sai: +# ops: +# insert_one_op: +# insert_one: "TEMPLATE(collection)" +# document: +# _id: "{seq_key}" +# value: "{seq_value}" +# main_read_with_text_sai: +# params: +# ratio: 5 +# ops: +# find_op: +# find: "TEMPLATE(collection)" +# filter: +# value: "{rw_value}" diff --git a/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java b/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java index 4babf270a..d65615c9b 100644 --- a/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java +++ b/nb-virtdata/virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java @@ -28,6 +28,22 @@ import static org.assertj.core.api.Assertions.assertThat; public class ParsedTemplateMapTest { +// @Test +// public void testBindpointTypes() { +// ParsedTemplateMap ptm = new ParsedTemplateMap( +// "name1", +// Map.of( +// "field1","{binding1}", +// "field2", " {binding2}" +// ), +// Map.of( +// "binding1","TestingStringFunc('b1')", +// "binding2","TestingStringFunc('b2')" +// ), +// List.of(Map.of())); +// +// } + @Test public void testParsedTemplateMap() { ParsedTemplateMap ptm = new ParsedTemplateMap("name1", Map.of("string1", "string2"), Map.of(), List.of()); From 12a7472932a64ab6384e4af6b26ef89fab6c309f Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 10 May 2024 19:56:54 -0500 Subject: [PATCH 19/19] passthru for correct vector types, op updates, runner update --- ...SCENARIO astra DAPI dapi_novector.run.xml} | 4 +- .../opdispensers/DataApiOpDispenser.java | 3 + .../resources/activities/astra_kv_dapi.yaml | 92 ++++++++++++------- 3 files changed, 63 insertions(+), 36 deletions(-) rename .run/{astra DAPI schema collection=baselines.run.xml => SCENARIO astra DAPI dapi_novector.run.xml} (66%) diff --git a/.run/astra DAPI schema collection=baselines.run.xml b/.run/SCENARIO astra DAPI dapi_novector.run.xml similarity index 66% rename from .run/astra DAPI schema collection=baselines.run.xml rename to .run/SCENARIO astra DAPI dapi_novector.run.xml index e50fb39a0..4c2231ff5 100644 --- a/.run/astra DAPI schema collection=baselines.run.xml +++ b/.run/SCENARIO astra DAPI dapi_novector.run.xml @@ -1,12 +1,12 @@ - +