From 6545156ce3ac123e31b21457e6cbc0832959c1eb Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 24 Apr 2024 09:16:35 -0400 Subject: [PATCH 01/15] minor updates to milvus driver files --- .../io/nosqlbench/adapter/milvus/MilvusSpace.java | 2 +- .../milvus/ops/MilvusCreateCollectionOp.java | 2 +- .../adapter-milvus/src/main/resources/milvus.md | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java index 0553126e4..d212c495e 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java @@ -45,7 +45,7 @@ public class MilvusSpace implements AutoCloseable { protected MilvusServiceClient client; - private final Map connections = new HashMap<>(); +// private final Map connections = new HashMap<>(); /** * Create a new MilvusSpace Object which stores all stateful contextual information needed to interact diff --git a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusCreateCollectionOp.java b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusCreateCollectionOp.java index a8d2bddda..164dc9fdd 100644 --- a/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusCreateCollectionOp.java +++ b/nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/ops/MilvusCreateCollectionOp.java @@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; public class MilvusCreateCollectionOp extends MilvusBaseOp { /** - * Create a new {@link ParsedOp} encapsulating a call to the Milvus/Zilliz client delete method + * Create a new {@link ParsedOp} encapsulating a call to the Milvus/Zilliz client create method. * * @param client The associated {@link MilvusServiceClient} used to communicate with the database * @param request The {@link CreateCollectionParam} built for this operation diff --git a/nb-adapters/adapter-milvus/src/main/resources/milvus.md b/nb-adapters/adapter-milvus/src/main/resources/milvus.md index 76e251d50..ed2563ba3 100644 --- a/nb-adapters/adapter-milvus/src/main/resources/milvus.md +++ b/nb-adapters/adapter-milvus/src/main/resources/milvus.md @@ -9,12 +9,15 @@ https://github.com/milvus-io/milvus-sdk-java. The following parameters must be supplied to the adapter at runtime in order to successfully connect to an instance of the Milvus/Zilliz database: -* token - In order to use the pinecone database you must have an account. Once the account is created you can [request +* `token` - In order to use the Milvus/Zilliz database you must have an account. Once the account is created you + can [request an api key/token](https://milvus.io/docs/users_and_roles.md#Users-and-Roles). This key will need to be provided any - time a - database connection is desired. -* uri - When an Index is created in the database the uri must be specified as well. The adapter will - use the default value of localhost:19530 if none is provided at runtime. + time a database connection is desired. Alternatively, + the api key can be stored in a file securely and referenced via the `token_file` config option pointing to the path of + the file. +* `uri` - When an index is created in the database the URI/endpoint must be specified as well. The adapter will + use the default value of `localhost:19530` if none is provided at runtime. +* `database_name` or `database` - the name of the database to use. For Zilliz, only `default` is supported. ## Op Templates From 053dbec8f5ea1dd5bcb623275d9f49f75fe3b17e Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Wed, 24 Apr 2024 09:23:58 -0400 Subject: [PATCH 02/15] Initial draft skeleton for Qdrant driver adapter --- nb-adapters/adapter-qdrant/pom.xml | 62 +++++++ .../adapter/qdrant/QdrantAdapterUtils.java | 73 +++++++++ .../adapter/qdrant/QdrantDriverAdapter.java | 54 +++++++ .../qdrant/QdrantDriverAdapterLoader.java | 32 ++++ .../adapter/qdrant/QdrantOpMapper.java | 66 ++++++++ .../adapter/qdrant/QdrantSpace.java | 137 ++++++++++++++++ .../opdispensers/QdrantBaseOpDispenser.java | 64 ++++++++ .../QdrantCreateCollectionOpDispenser.java | 153 ++++++++++++++++++ .../adapter/qdrant/ops/QdrantBaseOp.java | 75 +++++++++ .../qdrant/ops/QdrantCreateCollectionOp.java | 53 ++++++ .../adapter/qdrant/types/QdrantOpType.java | 21 +++ .../activities/qdrant_vectors_live.yaml | 53 ++++++ .../src/main/resources/qdrant.md | 97 +++++++++++ nb-adapters/nb-adapters-included/pom.xml | 14 ++ nb-adapters/pom.xml | 12 +- 15 files changed, 965 insertions(+), 1 deletion(-) create mode 100644 nb-adapters/adapter-qdrant/pom.xml create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantAdapterUtils.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapter.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapterLoader.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantSpace.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantBaseOp.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreateCollectionOp.java create mode 100644 nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/types/QdrantOpType.java create mode 100644 nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml create mode 100644 nb-adapters/adapter-qdrant/src/main/resources/qdrant.md diff --git a/nb-adapters/adapter-qdrant/pom.xml b/nb-adapters/adapter-qdrant/pom.xml new file mode 100644 index 000000000..639a3fdcf --- /dev/null +++ b/nb-adapters/adapter-qdrant/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + adapter-qdrant + jar + + + mvn-defaults + io.nosqlbench + ${revision} + ../../mvn-defaults + + + ${project.artifactId} + + An nosqlbench adapter driver module for the Qdrant database. + + + + + io.nosqlbench + nb-annotations + ${revision} + compile + + + io.nosqlbench + adapters-api + ${revision} + compile + + + io.grpc + grpc-protobuf + + + 1.59.0 + + + com.google.protobuf + protobuf-java-util + + + 3.24.0 + + + com.google.guava + guava + + + 30.1-jre + + + io.qdrant + client + 1.9.0 + + + diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantAdapterUtils.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantAdapterUtils.java new file mode 100644 index 000000000..07583d023 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantAdapterUtils.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; + +public class QdrantAdapterUtils { + + public static final String QDRANT = "qdrant"; + + public static List splitNames(String input) { + assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input); + return Arrays.stream(input.split("( +| *, *)")) + .filter(StringUtils::isNotBlank) + .toList(); + } + + public static List splitLongs(String input) { + assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input); + return Arrays.stream(input.split("( +| *, *)")) + .filter(StringUtils::isNotBlank) + .map(Long::parseLong) + .toList(); + } + + + /** + * Mask the digits in the given string with '*' + * + * @param unmasked The string to mask + * @return The masked string + */ + protected static String maskDigits(String unmasked) { + assert StringUtils.isNotBlank(unmasked) && StringUtils.isNotEmpty(unmasked); + int inputLength = unmasked.length(); + StringBuilder masked = new StringBuilder(inputLength); + for (char ch : unmasked.toCharArray()) { + if (Character.isDigit(ch)) { + masked.append("*"); + } else { + masked.append(ch); + } + } + return masked.toString(); + } + +// public static int[] intArrayFromMilvusSearchResults(String fieldName, R result) { +// SearchResultsWrapper wrapper = new SearchResultsWrapper(result.getData().getResults()); +// List fieldData = (List) wrapper.getFieldData(fieldName, 0); +// int[] indices = new int[fieldData.size()]; +// for (int i = 0; i < indices.length; i++) { +// indices[i] = Integer.parseInt(fieldData.get(i)); +// } +// return indices; +// } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapter.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapter.java new file mode 100644 index 000000000..49933789b --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapter.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant; + +import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; +import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.labels.NBLabels; + +import java.util.function.Function; + +import static io.nosqlbench.adapter.qdrant.QdrantAdapterUtils.QDRANT; + +@Service(value = DriverAdapter.class, selector = QDRANT) +public class QdrantDriverAdapter extends BaseDriverAdapter, QdrantSpace> { + + public QdrantDriverAdapter(NBComponent parentComponent, NBLabels labels) { + super(parentComponent, labels); + } + + @Override + public OpMapper> getOpMapper() { + return new QdrantOpMapper(this); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return (s) -> new QdrantSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(QdrantSpace.getConfigModel()); + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapterLoader.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapterLoader.java new file mode 100644 index 000000000..b861c65c5 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantDriverAdapterLoader.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant; + +import io.nosqlbench.adapter.diag.DriverAdapterLoader; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.labels.NBLabels; + +import static io.nosqlbench.adapter.qdrant.QdrantAdapterUtils.QDRANT; + +@Service(value = DriverAdapterLoader.class, selector = QDRANT) +public class QdrantDriverAdapterLoader implements DriverAdapterLoader { + @Override + public QdrantDriverAdapter load(NBComponent parent, NBLabels childLabels) { + return new QdrantDriverAdapter(parent, childLabels); + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java new file mode 100644 index 000000000..1440dba6b --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantOpMapper.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant; + +import io.nosqlbench.adapter.qdrant.opdispensers.QdrantBaseOpDispenser; +import io.nosqlbench.adapter.qdrant.opdispensers.QdrantCreateCollectionOpDispenser; +import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; +import io.nosqlbench.adapter.qdrant.types.QdrantOpType; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.engine.api.templating.TypeAndTarget; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class QdrantOpMapper implements OpMapper> { + private static final Logger logger = LogManager.getLogger(QdrantOpMapper.class); + private final QdrantDriverAdapter adapter; + + /** + * Create a new QdrantOpMapper implementing the {@link OpMapper} interface. + * + * @param adapter The associated {@link QdrantDriverAdapter} + */ + public QdrantOpMapper(QdrantDriverAdapter adapter) { + this.adapter = adapter; + } + + /** + * Given an instance of a {@link ParsedOp} returns the appropriate {@link QdrantBaseOpDispenser} subclass + * + * @param op The {@link ParsedOp} to be evaluated + * @return The correct {@link QdrantBaseOpDispenser} subclass based on the op type + */ + @Override + public OpDispenser> apply(ParsedOp op) { + TypeAndTarget typeAndTarget = op.getTypeAndTarget( + QdrantOpType.class, + String.class, + "type", + "target" + ); + logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); + + return switch (typeAndTarget.enumId) { +// case drop_collection -> new QdrantDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); + case create_collection -> new QdrantCreateCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction); +// default -> throw new RuntimeException("Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + +// "mapping parsed op " + op); + }; + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantSpace.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantSpace.java new file mode 100644 index 000000000..3cc19da29 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantSpace.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant; + +import io.nosqlbench.nb.api.config.standard.ConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.config.standard.Param; +import io.qdrant.client.QdrantClient; +import io.qdrant.client.QdrantGrpcClient; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; + +/** + * The {@code QdrantSpace} class is a context object which stores all stateful contextual information needed to interact + * with the Qdrant database instance. + * + * @see Qdrant cloud quick start guide + * @see Qdrant quick start guide + * @see Qdrant Java client + */ +public class QdrantSpace implements AutoCloseable { + private final static Logger logger = LogManager.getLogger(QdrantSpace.class); + private final String name; + private final NBConfiguration cfg; + + protected QdrantClient client; + +// private final Map connections = new HashMap<>(); + + /** + * Create a new QdrantSpace Object which stores all stateful contextual information needed to interact + * with the Qdrant database instance. + * + * @param name The name of this space + * @param cfg The configuration ({@link NBConfiguration}) for this nb run + */ + public QdrantSpace(String name, NBConfiguration cfg) { + this.name = name; + this.cfg = cfg; + } + + public synchronized QdrantClient getClient() { + if (client == null) { + client = createClient(); + } + return client; + } + + private QdrantClient createClient() { + String uri = cfg.get("uri"); + int grpcPort = cfg.getOptional("grpc_port").map(Integer::parseInt).orElse(6334); + boolean useTls = cfg.getOptional("use_tls").map(Boolean::parseBoolean).orElse(true); + + var builder = QdrantGrpcClient.newBuilder(uri, grpcPort, useTls); + var requiredToken = cfg.getOptional("token_file") + .map(Paths::get) + .map( + tokenFilePath -> { + try { + return Files.readAllLines(tokenFilePath).getFirst(); + } catch (IOException e) { + String error = "Error while reading token from file:" + tokenFilePath; + logger.error(error, e); + throw new RuntimeException(e); + } + } + ).orElseGet( + () -> cfg.getOptional("token") + .orElseThrow(() -> new RuntimeException("You must provide either a token_file or a token to " + + "configure a Qdrant client")) + ); + builder = builder.withApiKey(requiredToken); + builder = builder.withTimeout( + Duration.ofMillis(NumberUtils.toInt(cfg.getOptional("timeout_ms").orElse("3000"))) + ); + + logger.info("{}: Creating new Qdrant Client with (masked) token [{}], uri/endpoint [{}]", + this.name, QdrantAdapterUtils.maskDigits(requiredToken), cfg.get("uri").toString()); + return new QdrantClient(builder.build()); + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(QdrantSpace.class) + .add( + Param.optional("token_file", String.class, "the file to load the api token from") + ) + .add( + Param.defaultTo("token", "qdrant") + .setDescription("the Qdrant api token to use to connect to the database") + ) + .add( + Param.defaultTo("uri", "localhost") + .setDescription("the URI endpoint in which the database is running. Do not provide any suffix like https:// here.") + ) + .add( + Param.defaultTo("use_tls", true) + .setDescription("whether to use TLS for the connection. Defaults to true.") + ) + .add( + Param.defaultTo("timeout_ms", 3000) + .setDescription("sets the timeout in milliseconds for all requests. Defaults to 3000ms.") + ) + .add( + Param.defaultTo("grpc_port", 6443) + .setDescription("the port to use for the gRPC connection. Defaults to 6334.") + ) + .asReadOnly(); + } + + @Override + public void close() throws Exception { + if (client != null) { + client.close(); + } + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java new file mode 100644 index 000000000..fc3937c74 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant.opdispensers; + +import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter; +import io.nosqlbench.adapter.qdrant.QdrantSpace; +import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.qdrant.client.QdrantClient; + +import java.util.function.LongFunction; + +public abstract class QdrantBaseOpDispenser extends BaseOpDispenser, QdrantSpace> { + + protected final LongFunction qdrantSpaceFunction; + protected final LongFunction clientFunction; + private final LongFunction> opF; + private final LongFunction paramF; + + protected QdrantBaseOpDispenser(QdrantDriverAdapter adapter, ParsedOp op, LongFunction targetF) { + super((DriverAdapter)adapter, op); + this.qdrantSpaceFunction = adapter.getSpaceFunc(op); + this.clientFunction = (long l) -> this.qdrantSpaceFunction.apply(l).getClient(); + this.paramF = getParamFunc(this.clientFunction,op,targetF); + this.opF = createOpFunc(paramF, this.clientFunction, op, targetF); + } + protected QdrantDriverAdapter getDriverAdapter() { + return (QdrantDriverAdapter) adapter; + } + + public abstract LongFunction getParamFunc( + LongFunction clientF, + ParsedOp op, + LongFunction targetF + ); + + public abstract LongFunction> createOpFunc( + LongFunction paramF, + LongFunction clientF, + ParsedOp op, + LongFunction targetF + ); + + @Override + public QdrantBaseOp getOp(long value) { + return opF.apply(value); + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java new file mode 100644 index 000000000..e195308f6 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2020-2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant.opdispensers; + +import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter; +import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; +import io.nosqlbench.adapter.qdrant.ops.QdrantCreateCollectionOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.qdrant.client.QdrantClient; +import io.qdrant.client.grpc.Collections.CreateCollection; +import io.qdrant.client.grpc.Collections.Distance; +import io.qdrant.client.grpc.Collections.VectorParams; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.LongFunction; + +public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(QdrantCreateCollectionOpDispenser.class); + + /** + * Create a new QdrantCreateCollectionOpDispenser subclassed from {@link QdrantBaseOpDispenser}. + * + * @param adapter The associated {@link QdrantDriverAdapter} + * @param op The {@link ParsedOp} encapsulating the activity for this cycle + * @param targetFunction A LongFunction that returns the specified Qdrant Index for this Op + * @see Qdrant Create Collection. + */ + public QdrantCreateCollectionOpDispenser(QdrantDriverAdapter adapter, + ParsedOp op, + LongFunction targetFunction) { + super(adapter, op, targetFunction); + } + + @Override + public LongFunction getParamFunc( + LongFunction clientF, + ParsedOp op, + LongFunction targetF + ) { + LongFunction ebF = + l -> CreateCollection.newBuilder().setCollectionName(targetF.apply(l)); + +// LongFunction ebF = +// l -> CreateCollectionParam.newBuilder().withCollectionName(targetF.apply(l)); + + Map namedVectorsMap = buildNamedVectorsStruct( + op.getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey) + ); + +// ebF = op.enhanceFuncOptionally(ebF, "shards_num", Number.class, +// (VectorParams.Builder b, Number n) -> b.withShardsNum(n.intValue())); +// ebF = op.enhanceFuncOptionally(ebF, "partition_num", Number.class, +// (CreateCollectionParam.Builder b, Number n) -> b.withPartitionsNum(n.intValue())); +// ebF = op.enhanceFuncOptionally(ebF, "description", String.class, +// VectorParams.Builder::withDescription); +// ebF = op.enhanceEnumOptionally(ebF, "consistency_level", +// ConsistencyLevelEnum.class, CreateCollectionParam.Builder::withConsistencyLevel); +// ebF = op.enhanceFuncOptionally(ebF, "database_name", String.class, +// CreateCollectionParam.Builder::withDatabaseName); + +// List fieldTypes = buildFieldTypesStruct( +// op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey) +// ); + // TODO - HERE +// final LongFunction f = ebF; +// ebF = l -> f.apply(l).withSchema(CollectionSchemaParam.newBuilder().withFieldTypes(fieldTypes).build()); +// +// final LongFunction lastF = ebF; +// return l -> lastF.apply(l).build(); + return l -> ebF.apply(l).build(); + } + + private Map buildNamedVectorsStruct(Map namedVectorsData) { + Map namedVectors = new HashMap<>(); + namedVectorsData.forEach((name, fieldspec) -> { + VectorParams.Builder builder = VectorParams.newBuilder(); + // TODO - these are mandatory items; see how to achieve this. + fieldspec.getOptionalStaticConfig("distance", Distance.class) + .ifPresent(builder::setDistance); + fieldspec.getOptionalStaticConfig("size", Number.class) + .ifPresent((Number n) -> builder.setSize(n.intValue())); + + namedVectors.put(name, builder.build()); + }); + return namedVectors; + } + + // https://qdrant.tech/documentation/concepts/collections/#create-a-collection + @Override + public LongFunction> createOpFunc( + LongFunction paramF, + LongFunction clientF, + ParsedOp op, + LongFunction targetF + ) { + return l -> new QdrantCreateCollectionOp(clientF.apply(l), paramF.apply(l)); + } + + /** + * Function to build the {@link FieldType}s for the {@link VectorParams}. + * + * @param fieldTypesData The static map of config data from the create collection request + * @return a list of static field types + */ +// private List buildFieldTypesStruct(Map fieldTypesData) { +// List fieldTypes = new ArrayList<>(); +// fieldTypesData.forEach((name, fieldspec) -> { +// FieldType.Builder builder = FieldType.newBuilder() +// .withName(name); +// +// fieldspec.getOptionalStaticValue("primary_key", Boolean.class) +// .ifPresent(builder::withPrimaryKey); +// fieldspec.getOptionalStaticValue("auto_id", Boolean.class) +// .ifPresent(builder::withAutoID); +// fieldspec.getOptionalStaticConfig("max_length", Number.class) +// .ifPresent((Number n) -> builder.withMaxLength(n.intValue())); +// fieldspec.getOptionalStaticConfig("max_capacity", Number.class) +// .ifPresent((Number n) -> builder.withMaxCapacity(n.intValue())); +// fieldspec.getOptionalStaticValue(List.of("partition_key", "partition"), Boolean.class) +// .ifPresent(builder::withPartitionKey); +// fieldspec.getOptionalStaticValue("dimension", Number.class) +// .ifPresent((Number n) -> builder.withDimension(n.intValue())); +// fieldspec.getOptionalStaticConfig("data_type", String.class) +// .map(DataType::valueOf) +// .ifPresent(builder::withDataType); +// fieldspec.getOptionalStaticConfig("type_params", Map.class) +// .ifPresent(builder::withTypeParams); +// fieldspec.getOptionalStaticConfig("element_type", String.class) +// .map(DataType::valueOf) +// .ifPresent(builder::withElementType); +// +// fieldTypes.add(builder.build()); +// }); +// return fieldTypes; +// } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantBaseOp.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantBaseOp.java new file mode 100644 index 000000000..f99d4f0b2 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantBaseOp.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant.ops; + +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import io.qdrant.client.QdrantClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public abstract class QdrantBaseOp implements CycleOp { + + protected final static Logger logger = LogManager.getLogger(QdrantBaseOp.class); + + protected final QdrantClient client; + protected final T request; + protected final LongFunction apiCall; + + public QdrantBaseOp(QdrantClient client, T requestParam) { + this.client = client; + this.request = requestParam; + this.apiCall = this::applyOp; + } + + public QdrantBaseOp(QdrantClient client, T requestParam, LongFunction call) { + this.client = client; + this.request = requestParam; + this.apiCall = call; + } + + @Override + public final Object apply(long value) { + logger.trace("applying op: {}", this); + + try { + Object result = applyOp(value); +// if (result instanceof R r) { +// var error = r.getException(); +// if (error != null) { +// throw error; +// } +// } else { +// logger.warn("Op '" + this.toString() + "' did not return a Result 'R' type." + +// " Exception handling will be bypassed" +// ); +// } + return result; + } catch (Exception e) { + RuntimeException rte = (RuntimeException) e; + throw rte; + } + } + + public abstract Object applyOp(long value); + + @Override + public String toString() { + return "QdrantOp(" + this.request.getClass().getSimpleName() + ")"; + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreateCollectionOp.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreateCollectionOp.java new file mode 100644 index 000000000..4a0198a57 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreateCollectionOp.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant.ops; + +import com.google.common.util.concurrent.ListenableFuture; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.qdrant.client.QdrantClient; +import io.qdrant.client.grpc.Collections.CollectionOperationResponse; +import io.qdrant.client.grpc.Collections.CreateCollection; + +public class QdrantCreateCollectionOp extends QdrantBaseOp { + /** + * Create a new {@link ParsedOp} encapsulating a call to the Qdrant create collection method. + * + * @param client The associated {@link QdrantClient} used to communicate with the database + * @param request The {@link CreateCollection} built for this operation + */ + public QdrantCreateCollectionOp(QdrantClient client, CreateCollection request) { + super(client, request); + } + + @Override + public Object applyOp(long value) { +// ListenableFuture response = client.createCollectionAsync( +// CreateCollection.newBuilder() +// .setCollectionName("test") +// .setVectorsConfig(VectorsConfig.newBuilder() +// .setParams( +// VectorParams.newBuilder() +// .setDistance(Distance.Cosine) +// .setSize(25) +// .build() +// ).build() +// ).build() +// ); + ListenableFuture response = client.createCollectionAsync(request); + return response; + } +} diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/types/QdrantOpType.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/types/QdrantOpType.java new file mode 100644 index 000000000..f932fdbd3 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/types/QdrantOpType.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.qdrant.types; + +public enum QdrantOpType { + create_collection, +} diff --git a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml new file mode 100644 index 000000000..96719b8b1 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml @@ -0,0 +1,53 @@ +min_version: 5.21 +description: | + This is a template for live vector search testing. + Template Variables: + + schema: Install the schema required to run the test + rampup: Measure how long it takes to load a set of embeddings + search: Measure how the system responds to queries while it + is indexing recently ingested data. + search: Run vector search with a set of default (or overridden) parameters + In all of these phases, it is important to instance the metrics with distinct names. + Also, aggregates of recall should include total aggregate as well as a moving average. + +scenarios: + qdrant_vectors: + schema_collection: >- + run tags==block:schema_collection + errors===stop + cycles===UNDEF threads===UNDEF + uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + +params: + driver: qdrant + instrument: true + +bindings: + row_key: ToString() + row_key_batch: Mul(TEMPLATE(batch_size)L); ListSizedStepped(TEMPLATE(batch_size),long->ToString()); + # filetype=hdf5 for TEMPLATE(filetype,hdf5) + test_floatlist_hdf5: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/test"); + relevant_indices_hdf5: HdfFileToIntArray("testdata/TEMPLATE(dataset).hdf5", "/neighbors") + distance_floatlist_hdf5: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/distance") + train_floatlist_hdf5: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/train"); + train_floatlist_hdf5_batch: Mul(TEMPLATE(batch_size)L); ListSizedStepped(TEMPLATE(batch_size),HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/train")); + # filetype=fvec for TEMPLATE(filetype,fvec) + test_floatlist_fvec: FVecReader("testdata/TEMPLATE(dataset)_TEMPLATE(trainsize)_query_vectors.fvec"); + relevant_indices_fvec: IVecReader("testdata/TEMPLATE(dataset)_TEMPLATE(trainsize)_indices_query.ivec"); + distance_floatlist_fvec: FVecReader("testdata/TEMPLATE(dataset)_TEMPLATE(testsize)_distances_count.fvec",TEMPLATE(dimensions),0); + train_floatlist_fvec: FVecReader("testdata/TEMPLATE(dataset)_TEMPLATE(trainsize)_base_vectors.fvec",TEMPLATE(dimensions),0); + train_floatlist_fvec_batch: Mul(TEMPLATE(batch_size,10)L); ListSizedStepped(TEMPLATE(batch_size),FVecReader("testdata/TEMPLATE(dataset)_TEMPLATE(trainsize)_base_vectors.fvec",TEMPLATE(dimensions),0)); + +blocks: + schema_collection: + ops: + # https://qdrant.github.io/qdrant/redoc/index.html#tag/collections/operation/create_collection + create_col_op: + create_collection: "TEMPLATE(collection)" +# description: "TEMPLATE(desc,a simple qdrant vector collection)" +# consistency_level: "TEMPLATE(write_cl,BOUNDED)" + vectors: + value: + size: TEMPLATE(dimensions,25) + distance: TEMPLATE(similarity_function,cosine) diff --git a/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md b/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md new file mode 100644 index 000000000..2361e54b1 --- /dev/null +++ b/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md @@ -0,0 +1,97 @@ +# qdrant driver adapter + +The qdrant driver adapter is a nb adapter for the qdrant driver, an open source Java driver for connecting to and +performing operations on an instance of a Qdrant Vector database. The driver is hosted on GitHub at +https://github.com/qdrant/java-client. + +## activity parameters + +The following parameters must be supplied to the adapter at runtime in order to successfully connect to an +instance of the [Qdrant database](https://qdrant.tech/documentation): + +* `token` - In order to use the Qdrant database you must have an account. Once the account is created you can [request + an api key/token](https://qdrant.tech/documentation/cloud/authentication/). This key will need to be provided any + time a database connection is desired. Alternatively, the api key can be stored in a file securely and referenced via + the `token_file` config option pointing to the path of the file. +* `uri` - When a collection/index is created in the database the URI (aka endpoint) must be specified as well. The adapter will + use the default value of `localhost:6334` if none is provided at runtime. Remember to *not* provide the `https://` + suffix. +* `grpc_port` - the GRPC port used by the Qdrant database. Defaults to `6334`. +* `use_tls` - option to leverage TLS for the connection. Defaults to `true`. +* `timeout_ms` - sets the timeout in milliseconds for all requests. Defaults to `3000`ms. + +## Op Templates + +The Qdrant adapter supports [**all operations**](../java/io/nosqlbench/adapter/qdrant/ops) supported by the Java +driver published by Qdrant. The official Qdrant API reference can be found at +https://qdrant.github.io/java-client/io/qdrant/client/package-summary.html + +The operations include a full-fledged support for all the APIs available in the Qdrant Java driver. +The following are a couple high level API operations. + +# TODO - Below needs to be updated post driver development. +* Create Collection +* Create Index +* Drop Collection +* Drop Index +* Search (vectors) + +## Examples +```yaml +ops: + example_create_collection: + create_collection: "example_collection" + description: "https://qdrant.io/api-reference/java/v2.3.x/Collection/createCollection().md" + collection_name: "example_collection" + shards_num: 10 + consistency_level: BOUNDED # BOUNDED, SESSION, EVENTUAL + field_types: + field1: + primary_key: true # only for Int64 and VarChar types + description: "field description" + data_type: "Varchar" + # Bool, Int8, Int16, Int32, Int64, + # Float, Double, String, Varchar, BinaryVector, FloatVector + type_param: + example_param1: example_pvalue1 + dimension: 1024 # >0 + max_length: 1024 # for String only, >0 + auto_id: false # Generate primary key? + partition_key: false # Primary key cannot be the partition key too + field2: + primary_key: false + description: "vector column/field" + data_type: "FloatVector" + dimension: 3 + + # https://qdrant.io/api-reference/java/v2.3.x/Index/dropIndex().md + example_drop_index: + drop_index: "exampe_collection_idx_name" + database_name: "my_database" + collection_name: "example_collection"" + + # https://qdrant.io/api-reference/java/v2.3.x/Collection/dropCollection().md + example_drop_collection: + drop_collection: "example_collection" + database_name: "my_database" + + # https://qdrant.io/api-reference/java/v2.3.x/High-level%20API/insert().md + example_insert_op: + insert: "example_collection_name" + rows: + field1: "row_key" + field2: "[1.2, 3.4, 5.6]" + + # https://qdrant.io/api-reference/java/v2.3.x/High-level%20API/search().md + # https://qdrant.io/api-reference/java/v2.3.x/Query%20and%20Search/search().md + example_search: + search: "example_collection" + vector: "[-0.4, 0.3, 0.99]" + metric_type: "COSINE" + out_fields: + - field1 + - field2 + vector_field_name: "field2" + top_k: 100 + consistency_level: "EVENTUALLY" +``` diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 5c108177c..182283b08 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -238,6 +238,20 @@ + + + adapter-qdrant-include + + false + + + + io.nosqlbench + adapter-qdrant + ${revision} + + + diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index 08af473e8..04d0bfcd2 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -1,5 +1,5 @@ >>>: " + actual_indices); + // driver-agnostic function + relevancy.accept({relevant_indices_TEMPLATE(filetype)},actual_indices); + // because we are "verifying" although this needs to be reorganized + return true; count_vectors: ops: diff --git a/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md b/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md index 2361e54b1..155cc2c57 100644 --- a/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md +++ b/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md @@ -22,76 +22,20 @@ instance of the [Qdrant database](https://qdrant.tech/documentation): ## Op Templates -The Qdrant adapter supports [**all operations**](../java/io/nosqlbench/adapter/qdrant/ops) supported by the Java -driver published by Qdrant. The official Qdrant API reference can be found at +The Qdrant adapter supports [**all operations**](../java/io/nosqlbench/adapter/qdrant/ops) supported by the [Java +driver published by Qdrant](https://github.com/qdrant/java-client). The official Qdrant API reference can be found at https://qdrant.github.io/java-client/io/qdrant/client/package-summary.html -The operations include a full-fledged support for all the APIs available in the Qdrant Java driver. +The operations include a full-fledged support for key APIs available in the Qdrant Java driver. The following are a couple high level API operations. -# TODO - Below needs to be updated post driver development. * Create Collection -* Create Index +* Count Points * Drop Collection -* Drop Index -* Search (vectors) +* Search Points (vectors) ## Examples -```yaml -ops: - example_create_collection: - create_collection: "example_collection" - description: "https://qdrant.io/api-reference/java/v2.3.x/Collection/createCollection().md" - collection_name: "example_collection" - shards_num: 10 - consistency_level: BOUNDED # BOUNDED, SESSION, EVENTUAL - field_types: - field1: - primary_key: true # only for Int64 and VarChar types - description: "field description" - data_type: "Varchar" - # Bool, Int8, Int16, Int32, Int64, - # Float, Double, String, Varchar, BinaryVector, FloatVector - type_param: - example_param1: example_pvalue1 - dimension: 1024 # >0 - max_length: 1024 # for String only, >0 - auto_id: false # Generate primary key? - partition_key: false # Primary key cannot be the partition key too - field2: - primary_key: false - description: "vector column/field" - data_type: "FloatVector" - dimension: 3 - # https://qdrant.io/api-reference/java/v2.3.x/Index/dropIndex().md - example_drop_index: - drop_index: "exampe_collection_idx_name" - database_name: "my_database" - collection_name: "example_collection"" +Check out the [full example available here](activities/qdrant_vectors_live.yaml). - # https://qdrant.io/api-reference/java/v2.3.x/Collection/dropCollection().md - example_drop_collection: - drop_collection: "example_collection" - database_name: "my_database" - - # https://qdrant.io/api-reference/java/v2.3.x/High-level%20API/insert().md - example_insert_op: - insert: "example_collection_name" - rows: - field1: "row_key" - field2: "[1.2, 3.4, 5.6]" - - # https://qdrant.io/api-reference/java/v2.3.x/High-level%20API/search().md - # https://qdrant.io/api-reference/java/v2.3.x/Query%20and%20Search/search().md - example_search: - search: "example_collection" - vector: "[-0.4, 0.3, 0.99]" - metric_type: "COSINE" - out_fields: - - field1 - - field2 - vector_field_name: "field2" - top_k: 100 - consistency_level: "EVENTUALLY" -``` +--- From e2534120ec0d92296cf8fe9f4e86bba86de24536 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Fri, 10 May 2024 12:43:44 -0400 Subject: [PATCH 15/15] cleanup dead code --- .../QdrantCreateCollectionOpDispenser.java | 267 ------------------ .../QdrantSearchPointsOpDispenser.java | 25 -- .../qdrant/ops/QdrantCountPointsOp.java | 2 +- .../activities/qdrant_vectors_live.yaml | 10 +- 4 files changed, 6 insertions(+), 298 deletions(-) diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java index ef5b18d12..e9d162ed9 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java @@ -26,7 +26,6 @@ import io.qdrant.client.grpc.Collections.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -58,20 +57,10 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser ebF = l -> CreateCollection.newBuilder().setCollectionName(targetF.apply(l)); - // new code - incomplete LongFunction> namedVectorParamsMap = buildNamedVectorsStruct(op); final LongFunction namedVectorsF = ebF; ebF = l -> namedVectorsF.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap( VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap.apply(l)).build())); - // new code - incomplete - // old code -// Map namedVectorParamsMap1 = buildNamedVectorsStruct( -// op.getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey) -// ); -// final LongFunction namedVectorsF1 = ebF; -// ebF = l -> namedVectorsF1.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap( -// VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap1).build())); - // old code ebF = op.enhanceFuncOptionally(ebF, "on_disk_payload", Boolean.class, CreateCollection.Builder::setOnDiskPayload); @@ -92,9 +81,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser wcdF = buildWalConfigDiff(walF.get()); ebF = l -> wallFunc.apply(l).setWalConfig(wcdF.apply(l)); } -// WalConfigDiff walConfig = buildWalConfigDiff(op); -// final LongFunction walConfigF = ebF; -// ebF = l -> walConfigF.apply(l).setWalConfig(walConfig); Optional> optConDifF = op.getAsOptionalFunction("optimizers_config", Map.class); if (optConDifF.isPresent()) { @@ -103,10 +89,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser wallFunc.apply(l).setOptimizersConfig(ocdF.apply(l)); } -// OptimizersConfigDiff ocDiff = buildOptimizerConfigDiff(op); -// final LongFunction ocF = ebF; -// ebF = l -> ocF.apply(l).setOptimizersConfig(ocDiff); - Optional> hnswConfigDiffF = op.getAsOptionalFunction("hnsw_config", Map.class); if (hnswConfigDiffF.isPresent()) { final LongFunction hnswConfigF = ebF; @@ -114,21 +96,12 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser hnswConfigF.apply(l).setHnswConfig(hcdF.apply(l)); } -// HnswConfigDiff hnswConfigDiff = buildHnswConfigDiff(op); -// final LongFunction hnswConfigF = ebF; -// ebF = l -> hnswConfigF.apply(l).setHnswConfig(hnswConfigDiff); - Optional> quantConfigF = op.getAsOptionalFunction("quantization_config", Map.class); if (quantConfigF.isPresent()) { final LongFunction qConF = ebF; LongFunction qcDiffF = buildQuantizationConfig(quantConfigF.get()); ebF = l -> qConF.apply(l).setQuantizationConfig(qcDiffF.apply(l)); } -// QuantizationConfig qcDiff = buildQuantizationConfig(op); -// if (qcDiff != null) { -// final LongFunction qcConfigF = ebF; -// ebF = l -> qcConfigF.apply(l).setQuantizationConfig(qcDiff); -// } Optional> sparseVectorsF = op.getAsOptionalFunction("sparse_vectors", Map.class); if (sparseVectorsF.isPresent()) { @@ -136,59 +109,11 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser sparseVectorsMap = buildSparseVectorsStruct(sparseVectorsF.get()); ebF = l -> sparseVecF.apply(l).setSparseVectorsConfig(sparseVectorsMap.apply(l)); } -// if (op.isDefined("sparse_vectors")) { -// SparseVectorConfig sparseVectorsMap = buildSparseVectorsStruct( -// op.getAsSubOps("sparse_vectors", ParsedOp.SubOpNaming.SubKey) -// ); -// final LongFunction sparseVectorsF = ebF; -// ebF = l -> sparseVectorsF.apply(l).setSparseVectorsConfig(sparseVectorsMap); -// } final LongFunction lastF = ebF; return l -> lastF.apply(l).build(); } - /** - * Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}. - * - * @param op {@link ParsedOp} containing the optimizer config data. - * @return {@link OptimizersConfigDiff} containing the optimizer config data - */ - private OptimizersConfigDiff buildOptimizerConfigDiff(ParsedOp op) { - OptimizersConfigDiff.Builder ocDiffBuilder = OptimizersConfigDiff.newBuilder(); - op.getOptionalStaticValue("optimizers_config", Map.class).ifPresent(ocData -> { - if (ocData.isEmpty()) { - return; - } else { - if (ocData.containsKey("deleted_threshold")) { - ocDiffBuilder.setDeletedThreshold(((Number) ocData.get("deleted_threshold")).doubleValue()); - } - if (ocData.containsKey("vacuum_min_vector_number")) { - ocDiffBuilder.setVacuumMinVectorNumber(((Number) ocData.get("vacuum_min_vector_number")).longValue()); - } - if (ocData.containsKey("default_segment_number")) { - ocDiffBuilder.setDefaultSegmentNumber(((Number) ocData.get("default_segment_number")).longValue()); - } - if (ocData.containsKey("max_segment_size")) { - ocDiffBuilder.setMaxSegmentSize(((Number) ocData.get("max_segment_size")).longValue()); - } - if (ocData.containsKey("memmap_threshold")) { - ocDiffBuilder.setMemmapThreshold(((Number) ocData.get("memmap_threshold")).longValue()); - } - if (ocData.containsKey("indexing_threshold")) { - ocDiffBuilder.setIndexingThreshold(((Number) ocData.get("indexing_threshold")).longValue()); - } - if (ocData.containsKey(("flush_interval_sec"))) { - ocDiffBuilder.setFlushIntervalSec(((Number) ocData.get("flush_interval_sec")).longValue()); - } - if (ocData.containsKey("max_optimization_threads")) { - ocDiffBuilder.setMaxOptimizationThreads(((Number) ocData.get("max_optimization_threads")).intValue()); - } - } - }); - return ocDiffBuilder.build(); - } - /** * Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}. * @@ -229,30 +154,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser { - if (walConfigData.isEmpty()) { - return; - } else { - if (walConfigData.containsKey("wal_capacity_mb")) { - walConfigDiffBuilder.setWalCapacityMb(((Number) walConfigData.get("wal_capacity_mb")).longValue()); - } - if (walConfigData.containsKey("wal_segments_ahead")) { - walConfigDiffBuilder.setWalSegmentsAhead(((Number) walConfigData.get("wal_segments_ahead")).longValue()); - } - } - }); - return walConfigDiffBuilder.build(); - } - /** * Build the {@link WalConfigDiff} from the provided {@link ParsedOp}. * @@ -275,32 +176,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser} namedVectorsData - * @return {@link VectorParams} containing the named vectors - */ - @Deprecated - private Map buildNamedVectorsStruct(Map namedVectorsData) { - Map namedVectors = new HashMap<>(); - VectorParams.Builder builder = VectorParams.newBuilder(); - namedVectorsData.forEach((name, fieldSpec) -> { - builder.setDistanceValue(fieldSpec.getStaticValue("distance_value", Number.class).intValue()); - builder.setSize(fieldSpec.getStaticValue("size", Number.class).longValue()); - fieldSpec.getOptionalStaticValue("on_disk", Boolean.class) - .ifPresent(builder::setOnDisk); - fieldSpec.getOptionalStaticValue("datatype_value", Number.class) - .ifPresent((Number value) -> builder.setDatatypeValue(value.intValue())); - - builder.setHnswConfig(buildHnswConfigDiff(fieldSpec)); - builder.setQuantizationConfig(buildQuantizationConfig(fieldSpec)); - - namedVectors.put(name, builder.build()); - }); - return namedVectors; - } - /** * Only named vectors are supported at this time in this driver. * @@ -348,124 +223,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenserQuantization Config - */ - @Deprecated - private QuantizationConfig buildQuantizationConfig(ParsedOp fieldSpec) { - QuantizationConfig.Builder qcBuilder = QuantizationConfig.newBuilder(); - fieldSpec.getOptionalStaticValue("quantization_config", Map.class).ifPresent(qcData -> { - if (qcData.isEmpty()) { - return; - } else { - // TODO - Approach #1 - feels ugly - Arrays.asList("binary", "product", "scalar") - .forEach(key -> { - if (qcData.containsKey(key)) { - switch (key) { - case "binary": - BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder(); - Map binaryQCData = (Map) qcData.get("binary"); - if (null != binaryQCData && !binaryQCData.isEmpty()) { - if (binaryQCData.containsKey("always_ram")) { - binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram")); - } - } - qcBuilder.setBinary(binaryBuilder); - break; - case "product": - ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder(); - Map productQCData = (Map) qcData.get("product"); - if (null != productQCData && !productQCData.isEmpty()) { - // Mandatory field - productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram")); - // Optional field(s) below - if (productQCData.containsKey("compression")) { - productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression"))); - } - } - qcBuilder.setProduct(productBuilder); - break; - case "scalar": - ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder(); - Map scalarQCData = (Map) qcData.get("scalar"); - if (null != scalarQCData && !scalarQCData.isEmpty()) { - // Mandatory field - scalarBuilder.setType(QuantizationType.forNumber(((Number) scalarQCData.get("type")).intValue())); - // Optional field(s) below - if (scalarQCData.containsKey("always_ram")) { - scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram")); - } - if (scalarQCData.containsKey("quantile")) { - scalarBuilder.setQuantile(((Number) scalarQCData.get("quantile")).floatValue()); - } - } - qcBuilder.setScalar(scalarBuilder); - break; - } - } - }); - // TODO - Approach #2 - equally feels ugly too. -// if (qcData.containsKey("binary")) { -// if (qcData.containsKey("scalar") || qcData.containsKey("product")) { -// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config"); -// } -// BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder(); -// Map binaryQCData = (Map) qcData.get("binary"); -// if (null != binaryQCData && !binaryQCData.isEmpty()) { -// if (binaryQCData.containsKey("always_ram")) { -// binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram")); -// } -// } -// qcBuilder.setBinary(binaryBuilder); -// } else if (qcData.containsKey("product")) { -// if (qcData.containsKey("binary") || qcData.containsKey("scalar")) { -// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config"); -// } -// ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder(); -// Map productQCData = (Map) qcData.get("product"); -// if (null != productQCData && !productQCData.isEmpty()) { -// // Mandatory field -// productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram")); -// // Optional field(s) below -// if (productQCData.containsKey("compression")) { -// productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression"))); -// } -// } -// qcBuilder.setProduct(productBuilder); -// } else if (qcData.containsKey("scalar")) { -// if (qcData.containsKey("binary") || qcData.containsKey("product")) { -// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config"); -// } -// ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder(); -// Map scalarQCData = (Map) qcData.get("scalar"); -// if (null != scalarQCData && !scalarQCData.isEmpty()) { -// // Mandatory field -// scalarBuilder.setType(QuantizationType.valueOf((String) scalarQCData.get("type"))); -// // Optional field(s) below -// if (scalarQCData.containsKey("always_ram")) { -// scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram")); -// } -// if (scalarQCData.containsKey("quantile")) { -// scalarBuilder.setQuantile((Float) scalarQCData.get("quantile")); -// } -// } -// qcBuilder.setScalar(scalarBuilder); -// } - } - }); - - // The below check is required to avoid INVALID_ARGUMENT: Unable to convert quantization config - if (qcBuilder.hasBinary() || qcBuilder.hasProduct() || qcBuilder.hasScalar()) { - return qcBuilder.build(); - } - return null; - } - private LongFunction buildQuantizationConfig(LongFunction quantConfMapLongFunc) { return l -> this.buildQuantizationConfig(quantConfMapLongFunc.apply(l)); } @@ -592,30 +349,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser sparseVectorsData) { - SparseVectorConfig.Builder builder = SparseVectorConfig.newBuilder(); - sparseVectorsData.forEach((name, fieldSpec) -> { - SparseVectorParams.Builder svpBuilder = SparseVectorParams.newBuilder(); - SparseIndexConfig.Builder sicBuilder = SparseIndexConfig.newBuilder(); - - fieldSpec.getOptionalStaticValue("full_scan_threshold", Number.class) - .ifPresent((Number value) -> sicBuilder.setFullScanThreshold(value.intValue())); - fieldSpec.getOptionalStaticValue("on_disk", Boolean.class) - .ifPresent(sicBuilder::setOnDisk); - - svpBuilder.setIndex(sicBuilder); - builder.putMap(name, svpBuilder.build()); - }); - return builder.build(); - } - /** * Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}. * diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java index 715fde0cd..61a8419d8 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java @@ -216,29 +216,4 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser>> getVectorFieldsFunction(ParsedOp op, String namedVectors) { -// return l -> { -// if (!op.isDefined(namedVectors)) { -// return Optional.empty(); -// } -// List fields = op.get(namedVectors, 0L); -// if (fields == null) { -// fields = op.get(namedVectors, 0L); -// } -// return fields; -// }; - LongFunction rowF = op.getAsRequiredFunction(namedVectors, Object.class); - Object testObject = rowF.apply(0L); - LongFunction> rowsF = null; - if(testObject instanceof List list) { - if (list.isEmpty()) { - throw new OpConfigError("Unable to detect type of list object for empty list for op named '" + op.getName() + "'"); - } else if (list.get(0) instanceof Float) { - rowsF = l -> (List) rowF.apply(l); - } - } - return Optional.ofNullable(rowsF); - } } diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java index 1cd82aca8..48f8e0ba5 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java @@ -37,7 +37,7 @@ public class QdrantCountPointsOp extends QdrantBaseOp { request.getExact(), Duration.ofMinutes(5) // opinionated default of 5 minutes for timeout ).get(); - logger.info("Total vector points counted: {}", result); + logger.info("[QdrantCountPointsOp] Total vector points counted: {}", result); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } diff --git a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml index 53fa40ef0..547c15373 100644 --- a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml +++ b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml @@ -17,27 +17,27 @@ scenarios: run tags==block:delete_collection errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) schema_collection: >- run tags==block:schema_collection errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) rampup: >- run tags==block:rampup errors===warn,counter,retry cycles===TEMPLATE(train_cycles,TEMPLATE(trainsize,1000)) threads===TEMPLATE(train_threads,AUTO) - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) count_vectors: >- run tags==block:count_vectors errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) search_points: >- run tags==block:search_points errors===warn,counter cycles===TEMPLATE(testann_cycles,TEMPLATE(testsize,1000)) threads===TEMPLATE(testann_threads,AUTO) - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) params: driver: qdrant