mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2024-11-21 16:27:51 -06:00
working code
This commit is contained in:
Madhavan Sridharan
parent
bbdd641318
commit
309fd6f7cc
@ -6,7 +6,7 @@
|
||||
<option name="useCurrentConnection" value="false" />
|
||||
</extension>
|
||||
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.delete_collection dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --show-stacktraces --logs-max 5" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.delete_collection dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:qdrant,instance:vectors,vendor:qdrant_v191" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5" />
|
||||
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/qdrant" />
|
||||
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
|
||||
<method v="2" />
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
<option name="useCurrentConnection" value="false" />
|
||||
</extension>
|
||||
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.schema_collection dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --show-stacktraces --logs-max 5" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.schema_collection dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:qdrant,instance:vectors,vendor:qdrant_v191" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5" />
|
||||
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/qdrant" />
|
||||
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
|
||||
<method v="2" />
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
<option name="useCurrentConnection" value="false" />
|
||||
</extension>
|
||||
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.search_points dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 testann_cycles=100 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --show-stacktraces --logs-max 5" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.search_points dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:qdrant,instance:vectors,vendor:qdrant_v191" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --report-csv-to metrics --show-stacktraces --logs-max 5" />
|
||||
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/qdrant" />
|
||||
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
|
||||
<method v="2" />
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
<option name="useCurrentConnection" value="false" />
|
||||
</extension>
|
||||
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.rampup dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --show-stacktraces --logs-max 5" />
|
||||
<option name="PROGRAM_PARAMETERS" value="qdrant_vectors_live qdrant_vectors.rampup dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=1 qdranthost=ded78a51-8370-47d8-adb0-6147f0fcbba2.us-east4-0.gcp.cloud.qdrant.io token_file=./apikey grpc_port=6334 --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:qdrant,instance:vectors,vendor:qdrant_v191" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5" />
|
||||
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/qdrant" />
|
||||
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
|
||||
<method v="2" />
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.qdrant;
|
||||
|
||||
import io.qdrant.client.grpc.Points.ScoredPoint;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Arrays;
|
||||
@ -40,7 +41,6 @@ public class QdrantAdapterUtils {
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Mask the digits in the given string with '*'
|
||||
*
|
||||
@ -61,13 +61,7 @@ public class QdrantAdapterUtils {
|
||||
return masked.toString();
|
||||
}
|
||||
|
||||
// public static int[] intArrayFromMilvusSearchResults(String fieldName, R<SearchResults> result) {
|
||||
// SearchResultsWrapper wrapper = new SearchResultsWrapper(result.getData().getResults());
|
||||
// List<String> fieldData = (List<String>) wrapper.getFieldData(fieldName, 0);
|
||||
// int[] indices = new int[fieldData.size()];
|
||||
// for (int i = 0; i < indices.length; i++) {
|
||||
// indices[i] = Integer.parseInt(fieldData.get(i));
|
||||
// }
|
||||
// return indices;
|
||||
// }
|
||||
public static int[] searchPointsResponseIdNumToIntArray(List<ScoredPoint> response) {
|
||||
return response.stream().mapToInt(r -> ((Number) r.getId().getNum()).intValue()).toArray();
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter;
|
||||
import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp;
|
||||
import io.nosqlbench.adapter.qdrant.ops.QdrantCreateCollectionOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.nosqlbench.nb.api.errors.OpConfigError;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.grpc.Collections.*;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
@ -28,6 +29,7 @@ import org.apache.logging.log4j.Logger;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<CreateCollection> {
|
||||
@ -56,12 +58,20 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
LongFunction<CreateCollection.Builder> ebF =
|
||||
l -> CreateCollection.newBuilder().setCollectionName(targetF.apply(l));
|
||||
|
||||
Map<String, VectorParams> namedVectorParamsMap = buildNamedVectorsStruct(
|
||||
op.getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey)
|
||||
);
|
||||
// new code - incomplete
|
||||
LongFunction<Map<String, VectorParams>> namedVectorParamsMap = buildNamedVectorsStruct(op);
|
||||
final LongFunction<CreateCollection.Builder> namedVectorsF = ebF;
|
||||
ebF = l -> namedVectorsF.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
|
||||
VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap).build()));
|
||||
VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap.apply(l)).build()));
|
||||
// new code - incomplete
|
||||
// old code
|
||||
// Map<String, VectorParams> namedVectorParamsMap1 = buildNamedVectorsStruct(
|
||||
// op.getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey)
|
||||
// );
|
||||
// final LongFunction<CreateCollection.Builder> namedVectorsF1 = ebF;
|
||||
// ebF = l -> namedVectorsF1.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
|
||||
// VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap1).build()));
|
||||
// old code
|
||||
|
||||
ebF = op.enhanceFuncOptionally(ebF, "on_disk_payload", Boolean.class,
|
||||
CreateCollection.Builder::setOnDiskPayload);
|
||||
@ -76,32 +86,64 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
ebF = op.enhanceFuncOptionally(ebF, "sharding_method", String.class,
|
||||
(CreateCollection.Builder b, String s) -> b.setShardingMethod(ShardingMethod.valueOf(s)));
|
||||
|
||||
WalConfigDiff walConfig = buildWalConfigDiff(op);
|
||||
final LongFunction<CreateCollection.Builder> walConfigF = ebF;
|
||||
ebF = l -> walConfigF.apply(l).setWalConfig(walConfig);
|
||||
Optional<LongFunction<Map>> walF = op.getAsOptionalFunction("wal_config", Map.class);
|
||||
if (walF.isPresent()) {
|
||||
final LongFunction<CreateCollection.Builder> wallFunc = ebF;
|
||||
LongFunction<WalConfigDiff> wcdF = buildWalConfigDiff(walF.get());
|
||||
ebF = l -> wallFunc.apply(l).setWalConfig(wcdF.apply(l));
|
||||
}
|
||||
// WalConfigDiff walConfig = buildWalConfigDiff(op);
|
||||
// final LongFunction<CreateCollection.Builder> walConfigF = ebF;
|
||||
// ebF = l -> walConfigF.apply(l).setWalConfig(walConfig);
|
||||
|
||||
OptimizersConfigDiff ocDiff = buildOptimizerConfigDiff(op);
|
||||
final LongFunction<CreateCollection.Builder> ocF = ebF;
|
||||
ebF = l -> ocF.apply(l).setOptimizersConfig(ocDiff);
|
||||
|
||||
HnswConfigDiff hnswConfigDiff = buildHnswConfigDiff(op);
|
||||
final LongFunction<CreateCollection.Builder> hnswConfigF = ebF;
|
||||
ebF = l -> hnswConfigF.apply(l).setHnswConfig(hnswConfigDiff);
|
||||
|
||||
QuantizationConfig qcDiff = buildQuantizationConfig(op);
|
||||
if (qcDiff != null) {
|
||||
final LongFunction<CreateCollection.Builder> qcConfigF = ebF;
|
||||
ebF = l -> qcConfigF.apply(l).setQuantizationConfig(qcDiff);
|
||||
Optional<LongFunction<Map>> optConDifF = op.getAsOptionalFunction("optimizers_config", Map.class);
|
||||
if (optConDifF.isPresent()) {
|
||||
final LongFunction<CreateCollection.Builder> wallFunc = ebF;
|
||||
LongFunction<OptimizersConfigDiff> ocdF = buildOptimizerConfigDiff(optConDifF.get());
|
||||
ebF = l -> wallFunc.apply(l).setOptimizersConfig(ocdF.apply(l));
|
||||
}
|
||||
|
||||
if (op.isDefined("sparse_vectors")) {
|
||||
SparseVectorConfig sparseVectorsMap = buildSparseVectorsStruct(
|
||||
op.getAsSubOps("sparse_vectors", ParsedOp.SubOpNaming.SubKey)
|
||||
);
|
||||
final LongFunction<CreateCollection.Builder> sparseVectorsF = ebF;
|
||||
ebF = l -> sparseVectorsF.apply(l).setSparseVectorsConfig(sparseVectorsMap);
|
||||
// OptimizersConfigDiff ocDiff = buildOptimizerConfigDiff(op);
|
||||
// final LongFunction<CreateCollection.Builder> ocF = ebF;
|
||||
// ebF = l -> ocF.apply(l).setOptimizersConfig(ocDiff);
|
||||
|
||||
Optional<LongFunction<Map>> hnswConfigDiffF = op.getAsOptionalFunction("hnsw_config", Map.class);
|
||||
if (hnswConfigDiffF.isPresent()) {
|
||||
final LongFunction<CreateCollection.Builder> hnswConfigF = ebF;
|
||||
LongFunction<HnswConfigDiff> hcdF = buildHnswConfigDiff(hnswConfigDiffF.get());
|
||||
ebF = l -> hnswConfigF.apply(l).setHnswConfig(hcdF.apply(l));
|
||||
}
|
||||
|
||||
// HnswConfigDiff hnswConfigDiff = buildHnswConfigDiff(op);
|
||||
// final LongFunction<CreateCollection.Builder> hnswConfigF = ebF;
|
||||
// ebF = l -> hnswConfigF.apply(l).setHnswConfig(hnswConfigDiff);
|
||||
|
||||
Optional<LongFunction<Map>> quantConfigF = op.getAsOptionalFunction("quantization_config", Map.class);
|
||||
if (quantConfigF.isPresent()) {
|
||||
final LongFunction<CreateCollection.Builder> qConF = ebF;
|
||||
LongFunction<QuantizationConfig> qcDiffF = buildQuantizationConfig(quantConfigF.get());
|
||||
ebF = l -> qConF.apply(l).setQuantizationConfig(qcDiffF.apply(l));
|
||||
}
|
||||
// QuantizationConfig qcDiff = buildQuantizationConfig(op);
|
||||
// if (qcDiff != null) {
|
||||
// final LongFunction<CreateCollection.Builder> qcConfigF = ebF;
|
||||
// ebF = l -> qcConfigF.apply(l).setQuantizationConfig(qcDiff);
|
||||
// }
|
||||
|
||||
Optional<LongFunction<Map>> sparseVectorsF = op.getAsOptionalFunction("sparse_vectors", Map.class);
|
||||
if (sparseVectorsF.isPresent()) {
|
||||
final LongFunction<CreateCollection.Builder> sparseVecF = ebF;
|
||||
LongFunction<SparseVectorConfig> sparseVectorsMap = buildSparseVectorsStruct(sparseVectorsF.get());
|
||||
ebF = l -> sparseVecF.apply(l).setSparseVectorsConfig(sparseVectorsMap.apply(l));
|
||||
}
|
||||
// if (op.isDefined("sparse_vectors")) {
|
||||
// SparseVectorConfig sparseVectorsMap = buildSparseVectorsStruct(
|
||||
// op.getAsSubOps("sparse_vectors", ParsedOp.SubOpNaming.SubKey)
|
||||
// );
|
||||
// final LongFunction<CreateCollection.Builder> sparseVectorsF = ebF;
|
||||
// ebF = l -> sparseVectorsF.apply(l).setSparseVectorsConfig(sparseVectorsMap);
|
||||
// }
|
||||
|
||||
final LongFunction<CreateCollection.Builder> lastF = ebF;
|
||||
return l -> lastF.apply(l).build();
|
||||
}
|
||||
@ -147,12 +189,53 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
return ocDiffBuilder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}.
|
||||
*
|
||||
* @param ocdMapLongFunc {@link LongFunction<Map>} containing the optimizer config data.
|
||||
* @return {@link OptimizersConfigDiff} containing the optimizer config data
|
||||
*/
|
||||
private LongFunction<OptimizersConfigDiff> buildOptimizerConfigDiff(LongFunction<Map> ocdMapLongFunc) {
|
||||
return l -> {
|
||||
OptimizersConfigDiff.Builder ocDiffBuilder = OptimizersConfigDiff.newBuilder();
|
||||
ocdMapLongFunc.apply(l).forEach((key, value) -> {
|
||||
if (key.equals("deleted_threshold")) {
|
||||
ocDiffBuilder.setDeletedThreshold(((Number) value).doubleValue());
|
||||
}
|
||||
if (key.equals("vacuum_min_vector_number")) {
|
||||
ocDiffBuilder.setVacuumMinVectorNumber(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("default_segment_number")) {
|
||||
ocDiffBuilder.setDefaultSegmentNumber(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("max_segment_size")) {
|
||||
ocDiffBuilder.setMaxSegmentSize(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("memmap_threshold")) {
|
||||
ocDiffBuilder.setMemmapThreshold(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("indexing_threshold")) {
|
||||
ocDiffBuilder.setIndexingThreshold(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals(("flush_interval_sec"))) {
|
||||
ocDiffBuilder.setFlushIntervalSec(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("max_optimization_threads")) {
|
||||
ocDiffBuilder.setMaxOptimizationThreads(((Number) value).intValue());
|
||||
}
|
||||
}
|
||||
);
|
||||
return ocDiffBuilder.build();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link WalConfigDiff} from the provided {@link ParsedOp}.
|
||||
*
|
||||
* @param op {@link ParsedOp} containing the WAL config data.
|
||||
* @return {@link WalConfigDiff} containing the WAL config data
|
||||
*/
|
||||
@Deprecated
|
||||
private WalConfigDiff buildWalConfigDiff(ParsedOp op) {
|
||||
WalConfigDiff.Builder walConfigDiffBuilder = WalConfigDiff.newBuilder();
|
||||
op.getOptionalStaticValue("wal_config", Map.class).ifPresent(walConfigData -> {
|
||||
@ -170,12 +253,35 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
return walConfigDiffBuilder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link WalConfigDiff} from the provided {@link ParsedOp}.
|
||||
*
|
||||
* @param mapLongFunction {@link LongFunction<Map>} containing the WAL config data.
|
||||
* @return {@link LongFunction<WalConfigDiff>} containing the WAL config data
|
||||
*/
|
||||
private LongFunction<WalConfigDiff> buildWalConfigDiff(LongFunction<Map> mapLongFunction) {
|
||||
return l -> {
|
||||
WalConfigDiff.Builder walConfigDiffBuilder = WalConfigDiff.newBuilder();
|
||||
mapLongFunction.apply(l).forEach((key, value) -> {
|
||||
if (key.equals("wal_capacity_mb")) {
|
||||
walConfigDiffBuilder.setWalCapacityMb(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("wal_segments_ahead")) {
|
||||
walConfigDiffBuilder.setWalSegmentsAhead(((Number) value).longValue());
|
||||
}
|
||||
}
|
||||
);
|
||||
return walConfigDiffBuilder.build();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Only named vectors are supported at this time in this driver.
|
||||
*
|
||||
* @param {@link Map<String, ParsedOp>} namedVectorsData
|
||||
* @return {@link VectorParams} containing the named vectors
|
||||
*/
|
||||
@Deprecated
|
||||
private Map<String, VectorParams> buildNamedVectorsStruct(Map<String, ParsedOp> namedVectorsData) {
|
||||
Map<String, VectorParams> namedVectors = new HashMap<>();
|
||||
VectorParams.Builder builder = VectorParams.newBuilder();
|
||||
@ -195,6 +301,53 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
return namedVectors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Only named vectors are supported at this time in this driver.
|
||||
*
|
||||
* @param {@link ParsedOp} op
|
||||
* @return {@link LongFunction<Map<String, VectorParams>>} containing the named vectors
|
||||
*/
|
||||
private LongFunction<Map<String, VectorParams>> buildNamedVectorsStruct(ParsedOp op) {
|
||||
if (!op.isDefined("vectors")) {
|
||||
throw new OpConfigError("Must provide values for 'vectors' in 'create_collection' op");
|
||||
}
|
||||
Optional<LongFunction<Map>> baseFunc = op.getAsOptionalFunction("vectors", Map.class);
|
||||
return baseFunc.<LongFunction<Map<String, VectorParams>>>map(mapLongFunc -> l -> {
|
||||
Map<String, Object> nvMap = mapLongFunc.apply(l);
|
||||
Map<String, VectorParams> namedVectors = new HashMap<>();
|
||||
nvMap.forEach((name, value) -> {
|
||||
VectorParams.Builder builder = VectorParams.newBuilder();
|
||||
if (value instanceof Map) {
|
||||
((Map<String, Object>) value).forEach((innerKey, innerValue) -> {
|
||||
if (innerKey.equals("distance_value")) {
|
||||
builder.setDistanceValue(((Number) innerValue).intValue());
|
||||
}
|
||||
if (innerKey.equals("size")) {
|
||||
builder.setSize(((Number) innerValue).longValue());
|
||||
}
|
||||
if (innerKey.equals("on_disk")) {
|
||||
builder.setOnDisk((Boolean) innerValue);
|
||||
}
|
||||
if (innerKey.equals("datatype_value")) {
|
||||
builder.setDatatypeValue(((Number) innerValue).intValue());
|
||||
}
|
||||
if (innerKey.equals("hnsw_config")) {
|
||||
builder.setHnswConfig(buildHnswConfigDiff((Map<String, Object>) innerValue));
|
||||
}
|
||||
if (innerKey.equals("quantization_config")) {
|
||||
builder.setQuantizationConfig(buildQuantizationConfig((Map<String, Object>) innerValue));
|
||||
}
|
||||
});
|
||||
} else {
|
||||
throw new OpConfigError("Named vectors must be a Map<String, Map<String, Object>>, but got "
|
||||
+ value.getClass().getSimpleName() + " instead for the inner value");
|
||||
}
|
||||
namedVectors.put(name, builder.build());
|
||||
});
|
||||
return namedVectors;
|
||||
}).orElse(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link QuantizationConfig} from the provided {@link ParsedOp}.
|
||||
*
|
||||
@ -202,6 +355,7 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
* @return The {@link QuantizationConfig} built from the provided {@link ParsedOp}
|
||||
* @see <a href="https://qdrant.tech/documentation/guides/quantization/#setting-up-quantization-in-qdrant">Quantization Config</a>
|
||||
*/
|
||||
@Deprecated
|
||||
private QuantizationConfig buildQuantizationConfig(ParsedOp fieldSpec) {
|
||||
QuantizationConfig.Builder qcBuilder = QuantizationConfig.newBuilder();
|
||||
fieldSpec.getOptionalStaticValue("quantization_config", Map.class).ifPresent(qcData -> {
|
||||
@ -312,6 +466,58 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
return null;
|
||||
}
|
||||
|
||||
private LongFunction<QuantizationConfig> buildQuantizationConfig(LongFunction<Map> quantConfMapLongFunc) {
|
||||
return l -> this.buildQuantizationConfig(quantConfMapLongFunc.apply(l));
|
||||
}
|
||||
|
||||
private QuantizationConfig buildQuantizationConfig(Map<String, Object> quantConfMap) {
|
||||
QuantizationConfig.Builder qcBuilder = QuantizationConfig.newBuilder();
|
||||
quantConfMap.forEach((key, value) -> {
|
||||
switch (key) {
|
||||
case "binary" -> {
|
||||
BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder();
|
||||
Map<?, ?> binaryQCData = (Map<?, ?>) value;
|
||||
if (null != binaryQCData && !binaryQCData.isEmpty()) {
|
||||
if (binaryQCData.containsKey("always_ram")) {
|
||||
binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram"));
|
||||
}
|
||||
qcBuilder.setBinary(binaryBuilder);
|
||||
}
|
||||
}
|
||||
case "product" -> {
|
||||
ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder();
|
||||
Map<?, ?> productQCData = (Map<?, ?>) value;
|
||||
if (null != productQCData && !productQCData.isEmpty()) {
|
||||
// Mandatory field
|
||||
productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram"));
|
||||
// Optional field(s) below
|
||||
if (productQCData.containsKey("compression")) {
|
||||
productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression")));
|
||||
}
|
||||
qcBuilder.setProduct(productBuilder);
|
||||
}
|
||||
}
|
||||
case "scalar" -> {
|
||||
ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder();
|
||||
Map<?, ?> scalarQCData = (Map<?, ?>) value;
|
||||
if (null != scalarQCData && !scalarQCData.isEmpty()) {
|
||||
// Mandatory field
|
||||
scalarBuilder.setType(QuantizationType.forNumber(((Number) scalarQCData.get("type")).intValue()));
|
||||
// Optional field(s) below
|
||||
if (scalarQCData.containsKey("always_ram")) {
|
||||
scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram"));
|
||||
}
|
||||
if (scalarQCData.containsKey("quantile")) {
|
||||
scalarBuilder.setQuantile(((Number) scalarQCData.get("quantile")).floatValue());
|
||||
}
|
||||
qcBuilder.setScalar(scalarBuilder);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return qcBuilder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link HnswConfigDiff} from the provided {@link ParsedOp}.
|
||||
*
|
||||
@ -319,6 +525,7 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
* @return The {@link HnswConfigDiff} built from the provided {@link ParsedOp}
|
||||
* @see <a href="https://qdrant.tech/documentation/concepts/indexing/#vector-index">HNSW Config</a>
|
||||
*/
|
||||
@Deprecated
|
||||
private HnswConfigDiff buildHnswConfigDiff(ParsedOp fieldSpec) {
|
||||
HnswConfigDiff.Builder hnswConfigBuilder = HnswConfigDiff.newBuilder();
|
||||
fieldSpec.getOptionalStaticValue("hnsw_config", Map.class).ifPresent(hnswConfigData -> {
|
||||
@ -348,12 +555,50 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
return hnswConfigBuilder.build();
|
||||
}
|
||||
|
||||
private LongFunction<HnswConfigDiff> buildHnswConfigDiff(LongFunction<Map> hnswConfigDiffMapLongFunc) {
|
||||
return l -> this.buildHnswConfigDiff(hnswConfigDiffMapLongFunc.apply(l));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link HnswConfigDiff} from the provided {@link ParsedOp}.
|
||||
*
|
||||
* @param hnswConfigDiffMap The {@link Map<String, Object>} containing the hnsw config data
|
||||
* @return The {@link LongFunction<HnswConfigDiff>} built from the provided {@link ParsedOp}
|
||||
* @see <a href="https://qdrant.tech/documentation/concepts/indexing/#vector-index">HNSW Config</a>
|
||||
*/
|
||||
private HnswConfigDiff buildHnswConfigDiff(Map<String, Object> hnswConfigDiffMap) {
|
||||
HnswConfigDiff.Builder hnswConfigBuilder = HnswConfigDiff.newBuilder();
|
||||
hnswConfigDiffMap.forEach((key, value) -> {
|
||||
if (key.equals("ef_construct")) {
|
||||
hnswConfigBuilder.setEfConstruct(((Number) value).longValue());
|
||||
}
|
||||
if (key.equals("m")) {
|
||||
hnswConfigBuilder.setM(((Number) value).intValue());
|
||||
}
|
||||
if (key.equals("full_scan_threshold")) {
|
||||
hnswConfigBuilder.setFullScanThreshold(((Number) value).intValue());
|
||||
}
|
||||
if (key.equals("max_indexing_threads")) {
|
||||
hnswConfigBuilder.setMaxIndexingThreads(((Number) value).intValue());
|
||||
}
|
||||
if (key.equals("on_disk")) {
|
||||
hnswConfigBuilder.setOnDisk((Boolean) value);
|
||||
}
|
||||
if (key.equals("payload_m")) {
|
||||
hnswConfigBuilder.setPayloadM(((Number) value).intValue());
|
||||
}
|
||||
}
|
||||
);
|
||||
return hnswConfigBuilder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}.
|
||||
*
|
||||
* @param sparseVectorsData The {@link ParsedOp} containing the sparse vectors data
|
||||
* @return The {@link SparseVectorConfig} built from the provided {@link ParsedOp}
|
||||
*/
|
||||
@Deprecated
|
||||
private SparseVectorConfig buildSparseVectorsStruct(Map<String, ParsedOp> sparseVectorsData) {
|
||||
SparseVectorConfig.Builder builder = SparseVectorConfig.newBuilder();
|
||||
sparseVectorsData.forEach((name, fieldSpec) -> {
|
||||
@ -371,6 +616,39 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}.
|
||||
*
|
||||
* @param sparseVectorsMapLongFunc The {@link LongFunction<Map>} containing the sparse vectors data
|
||||
* @return The {@link LongFunction<SparseVectorConfig>} built from the provided {@link ParsedOp}'s data
|
||||
*/
|
||||
private LongFunction<SparseVectorConfig> buildSparseVectorsStruct(LongFunction<Map> sparseVectorsMapLongFunc) {
|
||||
return l -> {
|
||||
SparseVectorConfig.Builder builder = SparseVectorConfig.newBuilder();
|
||||
sparseVectorsMapLongFunc.apply(l).forEach((key, value) -> {
|
||||
SparseVectorParams.Builder svpBuilder = SparseVectorParams.newBuilder();
|
||||
SparseIndexConfig.Builder sicBuilder = SparseIndexConfig.newBuilder();
|
||||
if (value instanceof Map) {
|
||||
((Map<String, Object>) value).forEach((innerKey, innerValue) -> {
|
||||
if (innerKey.equals("full_scan_threshold")) {
|
||||
sicBuilder.setFullScanThreshold(((Number) innerValue).intValue());
|
||||
}
|
||||
if (innerKey.equals("on_disk")) {
|
||||
sicBuilder.setOnDisk((Boolean) innerValue);
|
||||
}
|
||||
svpBuilder.setIndex(sicBuilder);
|
||||
builder.putMap((String) key, svpBuilder.build());
|
||||
}
|
||||
);
|
||||
} else {
|
||||
throw new OpConfigError("Sparse Vectors must be a Map<String, Map<String, Object>>, but got "
|
||||
+ value.getClass().getSimpleName() + " instead for the inner value");
|
||||
}
|
||||
});
|
||||
return builder.build();
|
||||
};
|
||||
}
|
||||
|
||||
// https://qdrant.tech/documentation/concepts/collections/#create-a-collection
|
||||
@Override
|
||||
public LongFunction<QdrantBaseOp<CreateCollection>> createOpFunc(
|
||||
|
||||
@ -19,12 +19,17 @@ package io.nosqlbench.adapter.qdrant.opdispensers;
|
||||
import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter;
|
||||
import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp;
|
||||
import io.nosqlbench.adapter.qdrant.ops.QdrantSearchPointsOp;
|
||||
import io.nosqlbench.adapter.qdrant.pojo.SearchPointsHelper;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.nosqlbench.nb.api.errors.OpConfigError;
|
||||
import io.qdrant.client.QdrantClient;
|
||||
import io.qdrant.client.ShardKeySelectorFactory;
|
||||
import io.qdrant.client.WithPayloadSelectorFactory;
|
||||
import io.qdrant.client.WithVectorsSelectorFactory;
|
||||
import io.qdrant.client.grpc.Points.*;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
@ -49,36 +54,170 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser<SearchP
|
||||
LongFunction<SearchPoints.Builder> ebF =
|
||||
l -> SearchPoints.newBuilder().setCollectionName(targetF.apply(l));
|
||||
|
||||
// query params here
|
||||
ebF = op.enhanceFuncOptionally(ebF, "timeout", Number.class,
|
||||
(SearchPoints.Builder b, Number t) -> b.setTimeout(t.longValue()));
|
||||
Optional<LongFunction<Object>> optionalConsistencyF = op.getAsOptionalFunction("consistency", Object.class);
|
||||
if (optionalConsistencyF.isPresent()) {
|
||||
LongFunction<SearchPoints.Builder> consistencyFunc = ebF;
|
||||
LongFunction<ReadConsistency> builtConsistency = buildReadConsistency(optionalConsistencyF.get());
|
||||
ebF = l -> consistencyFunc.apply(l).setReadConsistency(builtConsistency.apply(l));
|
||||
}
|
||||
|
||||
// body params here
|
||||
// - required items
|
||||
ebF = op.enhanceFuncOptionally(ebF, "limit", Number.class,
|
||||
(SearchPoints.Builder b, Number n) -> b.setLimit(n.longValue()));
|
||||
ebF = op.enhanceFuncOptionally(ebF, "vector_name", String.class, SearchPoints.Builder::setVectorName);
|
||||
ebF = op.enhanceFuncOptionally(ebF, "with_payload", Boolean.class,
|
||||
(SearchPoints.Builder b, Boolean wp) -> b.setWithPayload(WithPayloadSelector.newBuilder().setEnable(wp).build()));
|
||||
ebF = op.enhanceFuncOptionally(ebF, "with_vector", Boolean.class,
|
||||
(SearchPoints.Builder b, Boolean wp) -> b.setWithVectors(WithVectorsSelector.newBuilder().setEnable(wp).build()));
|
||||
ebF = op.enhanceFuncOptionally(ebF, "read_consistency", Number.class,
|
||||
(SearchPoints.Builder b, Number rc) -> b.setReadConsistency(
|
||||
ReadConsistency.newBuilder().setType(ReadConsistencyType.forNumber(rc.intValue())).build()));
|
||||
// ebF = op.enhanceFunc(ebF, List.of("vector_vector", "vectors"), List.class,
|
||||
// (SearchPoints.Builder b, List<Float> vec) -> b.addAllVector(vec));
|
||||
|
||||
Optional<LongFunction<List<Float>>> optionalVectorsF = getVectorFieldsFunction(op, "vector_vector");
|
||||
if(optionalVectorsF.isPresent()) {
|
||||
var rf = optionalVectorsF.get();
|
||||
LongFunction<SearchPoints.Builder> finalF2 = ebF;
|
||||
ebF = l -> finalF2.apply(l).addAllVector(rf.apply(l));
|
||||
}//ccvx .getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey)
|
||||
// );
|
||||
// final LongFunction<Collections.CreateCollection.Builder> namedVectorsF = ebF;
|
||||
// ebF = l -> namedVectorsF.apply(l).setVectorsConfig(Collections.VectorsConfig.newBuilder().setParams(namedVectorsMap));
|
||||
else {
|
||||
throw new OpConfigError("Must provide values for vectors");
|
||||
LongFunction<SearchPointsHelper> searchPointsHelperF = buildVectorForSearch(op);
|
||||
final LongFunction<SearchPoints.Builder> detailsOfNamedVectorsF = ebF;
|
||||
ebF = l -> detailsOfNamedVectorsF.apply(l)
|
||||
.setVectorName(searchPointsHelperF.apply(l).getVectorName())
|
||||
.addAllVector(searchPointsHelperF.apply(l).getVectorValues());
|
||||
//.setSparseIndices(searchPointsHelperF.apply(l).getSparseIndices()); throws NPE at their driver and hence below
|
||||
final LongFunction<SearchPoints.Builder> sparseIndicesF = ebF;
|
||||
ebF = l -> {
|
||||
SearchPoints.Builder builder = sparseIndicesF.apply(l);
|
||||
if (searchPointsHelperF.apply(l).getSparseIndices() != null) {
|
||||
builder.setSparseIndices(searchPointsHelperF.apply(l).getSparseIndices());
|
||||
}
|
||||
return builder;
|
||||
};
|
||||
|
||||
// - optional items
|
||||
ebF = op.enhanceFuncOptionally(ebF, "shard_key", String.class, (SearchPoints.Builder b, String sk) ->
|
||||
b.setShardKeySelector(ShardKeySelectorFactory.shardKeySelector(sk)));
|
||||
ebF = op.enhanceFuncOptionally(ebF, "score_threshold", Number.class,
|
||||
(SearchPoints.Builder b, Number n) -> b.setScoreThreshold(n.floatValue()));
|
||||
ebF = op.enhanceFuncOptionally(ebF, "offset", Number.class,
|
||||
(SearchPoints.Builder b, Number n) -> b.setOffset(n.longValue()));
|
||||
|
||||
Optional<LongFunction<Object>> optionalWithPayloadF = op.getAsOptionalFunction("with_payload", Object.class);
|
||||
if (optionalWithPayloadF.isPresent()) {
|
||||
LongFunction<SearchPoints.Builder> withPayloadFunc = ebF;
|
||||
LongFunction<WithPayloadSelector> builtWithPayload = buildWithPayloadSelector(optionalWithPayloadF.get());
|
||||
ebF = l -> withPayloadFunc.apply(l).setWithPayload(builtWithPayload.apply(l));
|
||||
}
|
||||
|
||||
Optional<LongFunction<Object>> optionalWithVectorF = op.getAsOptionalFunction("with_vector", Object.class);
|
||||
if (optionalWithVectorF.isPresent()) {
|
||||
LongFunction<SearchPoints.Builder> withVectorFunc = ebF;
|
||||
LongFunction<WithVectorsSelector> builtWithVector = buildWithVectorSelector(optionalWithVectorF.get());
|
||||
ebF = l -> withVectorFunc.apply(l).setWithVectors(builtWithVector.apply(l));
|
||||
}
|
||||
|
||||
// TODO - Implement filter, params
|
||||
|
||||
final LongFunction<SearchPoints.Builder> lastF = ebF;
|
||||
return l -> lastF.apply(l).build();
|
||||
}
|
||||
|
||||
private LongFunction<SearchPointsHelper> buildVectorForSearch(ParsedOp op) {
|
||||
if (!op.isDefined("vector")) {
|
||||
throw new OpConfigError("Must provide values for 'vector'");
|
||||
}
|
||||
Optional<LongFunction<List>> baseFunc = op.getAsOptionalFunction("vector", List.class);
|
||||
return baseFunc.<LongFunction<SearchPointsHelper>>map(listLongFunction -> l -> {
|
||||
List<Map<String, Object>> vectorPointsList = listLongFunction.apply(l);
|
||||
SearchPointsHelper searchPointsHelperBuilder = new SearchPointsHelper();
|
||||
vectorPointsList.forEach(point -> {
|
||||
if (point.containsKey("name")) {
|
||||
searchPointsHelperBuilder.setVectorName((String) point.get("name"));
|
||||
} else {
|
||||
throw new OpConfigError("Must provide values for 'name' within 'vector' field");
|
||||
}
|
||||
if (point.containsKey("values")) {
|
||||
searchPointsHelperBuilder.setVectorValues((List<Float>) point.get("values"));
|
||||
} else {
|
||||
throw new OpConfigError("Must provide values for 'values' within 'vector' field");
|
||||
}
|
||||
if (point.containsKey("sparse_indices")) {
|
||||
searchPointsHelperBuilder.setSparseIndices(
|
||||
SparseIndices.newBuilder().addAllData((List<Integer>) point.get("sparse_indices")).build());
|
||||
}
|
||||
});
|
||||
return searchPointsHelperBuilder;
|
||||
}).orElse(null);
|
||||
}
|
||||
|
||||
private LongFunction<WithVectorsSelector> buildWithVectorSelector(LongFunction<Object> objectLongFunction) {
|
||||
return l -> {
|
||||
Object withVector = objectLongFunction.apply(l);
|
||||
switch (withVector) {
|
||||
case Boolean b -> {
|
||||
return WithVectorsSelectorFactory.enable(b);
|
||||
}
|
||||
case List<?> objects when objects.getFirst() instanceof String -> {
|
||||
return WithVectorsSelectorFactory.include((List<String>) withVector);
|
||||
}
|
||||
case null, default -> {
|
||||
assert withVector != null;
|
||||
throw new OpConfigError("Invalid type for with_vector specified [{}]" +
|
||||
withVector.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private LongFunction<WithPayloadSelector> buildWithPayloadSelector(LongFunction<Object> objectLongFunction) {
|
||||
return l -> {
|
||||
Object withPayload = objectLongFunction.apply(l);
|
||||
switch (withPayload) {
|
||||
case Boolean b -> {
|
||||
return WithPayloadSelector.newBuilder().setEnable(b).build();
|
||||
}
|
||||
case List<?> objects when objects.getFirst() instanceof String -> {
|
||||
return WithPayloadSelectorFactory.include((List<String>) withPayload);
|
||||
}
|
||||
case Map<?, ?> map -> {
|
||||
WithPayloadSelector.Builder withPayloadSelector = WithPayloadSelector.newBuilder();
|
||||
map.forEach((key, value) -> {
|
||||
if (key.equals("include")) {
|
||||
withPayloadSelector.setInclude(
|
||||
PayloadIncludeSelector.newBuilder().addAllFields((List<String>) value).build());
|
||||
} else if (key.equals("exclude")) {
|
||||
withPayloadSelector.setExclude(
|
||||
PayloadExcludeSelector.newBuilder().addAllFields((List<String>) value).build());
|
||||
} else {
|
||||
throw new OpConfigError("Only 'include' & 'exclude' fields for with_payload map is supported," +
|
||||
" but we got [{}]" + key);
|
||||
}
|
||||
});
|
||||
|
||||
return withPayloadSelector.build();
|
||||
}
|
||||
case null, default -> {
|
||||
assert withPayload != null;
|
||||
throw new OpConfigError("Invalid type for with_payload specified [{}]" +
|
||||
withPayload.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param objectLongFunction the {@link LongFunction<Object>} from which the consistency for search will be built.
|
||||
* @return a {@link ReadConsistency} function object to be added to a Qdrant {@link UpsertPoints} request.
|
||||
* <p>
|
||||
* This method interrogates the subsection of the ParsedOp defined for vector parameters and constructs a list of
|
||||
* vector (dense plus sparse) points based on the included values, or returns null if this section is not populated.
|
||||
* The base function returns either the List of vectors or null, while the interior function builds the vectors
|
||||
* with a Builder pattern based on the values contained in the source ParsedOp.
|
||||
*/
|
||||
private LongFunction<ReadConsistency> buildReadConsistency(LongFunction<Object> objectLongFunction) {
|
||||
return l -> {
|
||||
Object consistency = objectLongFunction.apply(l);
|
||||
if (consistency instanceof Number) {
|
||||
return ReadConsistency.newBuilder().setTypeValue((Integer) consistency).build();
|
||||
} else if (consistency instanceof String) {
|
||||
return ReadConsistency.newBuilder().setType(ReadConsistencyType.valueOf((String) consistency)).build();
|
||||
} else {
|
||||
throw new OpConfigError("Invalid type for read consistency specified");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
private Optional<LongFunction<List<Float>>> getVectorFieldsFunction(ParsedOp op, String namedVectors) {
|
||||
// return l -> {
|
||||
// if (!op.isDefined(namedVectors)) {
|
||||
|
||||
@ -49,16 +49,6 @@ public abstract class QdrantBaseOp<T> implements CycleOp<Object> {
|
||||
|
||||
try {
|
||||
Object result = applyOp(value);
|
||||
// if (result instanceof R<?> r) {
|
||||
// var error = r.getException();
|
||||
// if (error != null) {
|
||||
// throw error;
|
||||
// }
|
||||
// } else {
|
||||
// logger.warn("Op '" + this.toString() + "' did not return a Result 'R' type." +
|
||||
// " Exception handling will be bypassed"
|
||||
// );
|
||||
// }
|
||||
return result;
|
||||
} catch (Exception e) {
|
||||
RuntimeException rte = (RuntimeException) e;
|
||||
|
||||
@ -30,12 +30,18 @@ public class QdrantSearchPointsOp extends QdrantBaseOp<SearchPoints> {
|
||||
|
||||
@Override
|
||||
public Object applyOp(long value) {
|
||||
List<ScoredPoint> result = null;
|
||||
List<ScoredPoint> response = null;
|
||||
try {
|
||||
result = client.searchAsync(request).get();
|
||||
logger.debug("[QdrantSearchPointsOp] Cycle {} has request: {}", value, request.toString());
|
||||
response = client.searchAsync(request).get();
|
||||
if (logger.isDebugEnabled()) {
|
||||
response.forEach(scoredPoint -> {
|
||||
logger.debug("[QdrantSearchPointsOp] Scored Point: {}", scoredPoint.toString());
|
||||
});
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return result;
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (c) 2020-2024 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.adapter.qdrant.pojo;
|
||||
|
||||
import io.qdrant.client.grpc.Points.SparseIndices;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Helper class to store the vector name, vector values and sparse indices to be used for searching points.
|
||||
*/
|
||||
public class SearchPointsHelper {
|
||||
private String vectorName;
|
||||
private List<Float> vectorValues;
|
||||
private SparseIndices sparseIndices;
|
||||
|
||||
public SearchPointsHelper(String vectorName, List<Float> vectorValues, SparseIndices sparseIndices) {
|
||||
this.vectorName = vectorName;
|
||||
this.vectorValues = vectorValues;
|
||||
this.sparseIndices = sparseIndices;
|
||||
}
|
||||
|
||||
public SearchPointsHelper() {
|
||||
}
|
||||
|
||||
public String getVectorName() {
|
||||
return vectorName;
|
||||
}
|
||||
|
||||
public void setVectorName(String vectorName) {
|
||||
this.vectorName = vectorName;
|
||||
}
|
||||
|
||||
public List<Float> getVectorValues() {
|
||||
return vectorValues;
|
||||
}
|
||||
|
||||
public void setVectorValues(List<Float> vectorValues) {
|
||||
this.vectorValues = vectorValues;
|
||||
}
|
||||
|
||||
public SparseIndices getSparseIndices() {
|
||||
return sparseIndices;
|
||||
}
|
||||
|
||||
public void setSparseIndices(SparseIndices sparseIndices) {
|
||||
this.sparseIndices = sparseIndices;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
SearchPointsHelper that = (SearchPointsHelper) o;
|
||||
return getVectorName().equals(that.getVectorName()) && getVectorValues().equals(that.getVectorValues()) && Objects.equals(getSparseIndices(), that.getSparseIndices());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = getVectorName().hashCode();
|
||||
result = 31 * result + getVectorValues().hashCode();
|
||||
result = 31 * result + Objects.hashCode(getSparseIndices());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -144,14 +144,40 @@ blocks:
|
||||
ops:
|
||||
search_points_op:
|
||||
search_points: "TEMPLATE(collection)"
|
||||
vector_name: "value"
|
||||
vector_vector: "{test_floatlist_TEMPLATE(filetype)}"
|
||||
limit: TEMPLATE(select_limit,100)
|
||||
with_payload: true
|
||||
with_vector: true
|
||||
timeout: 300 # 5 minutes
|
||||
# https://github.com/qdrant/qdrant/blob/v1.9.0/lib/api/src/grpc/proto/points.proto#L21-L25
|
||||
# 0 - All, 1 - Majority, 2 - Quorum
|
||||
read_consistency: 2
|
||||
consistency: "Quorum"
|
||||
with_payload: true
|
||||
with_vector: true
|
||||
limit: TEMPLATE(select_limit,100)
|
||||
# Another option to set with payload is as follows
|
||||
# with_payload: ["key1"]
|
||||
# Another option to set with payload is as follows
|
||||
# with_payload:
|
||||
# include: ["key1"]
|
||||
# exclude: ["key2"]
|
||||
vector:
|
||||
- name: "value"
|
||||
values: "{test_floatlist_TEMPLATE(filetype)}"
|
||||
#indices: "[1,7]"
|
||||
verifier-init: |
|
||||
relevancy= new io.nosqlbench.nb.api.engine.metrics.wrappers.RelevancyMeasures(_parsed_op);
|
||||
for (int k in List.of(100)) {
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("recall",k));
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("precision",k));
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("F1",k));
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("RR",k));
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("AP",k));
|
||||
}
|
||||
verifier: |
|
||||
// driver-specific function
|
||||
actual_indices=io.nosqlbench.adapter.qdrant.QdrantAdapterUtils.searchPointsResponseIdNumToIntArray(result)
|
||||
// System.out.println("actual_indices ------>>>>: " + actual_indices);
|
||||
// driver-agnostic function
|
||||
relevancy.accept({relevant_indices_TEMPLATE(filetype)},actual_indices);
|
||||
// because we are "verifying" although this needs to be reorganized
|
||||
return true;
|
||||
|
||||
count_vectors:
|
||||
ops:
|
||||
|
||||
@ -22,76 +22,20 @@ instance of the [Qdrant database](https://qdrant.tech/documentation):
|
||||
|
||||
## Op Templates
|
||||
|
||||
The Qdrant adapter supports [**all operations**](../java/io/nosqlbench/adapter/qdrant/ops) supported by the Java
|
||||
driver published by Qdrant. The official Qdrant API reference can be found at
|
||||
The Qdrant adapter supports [**all operations**](../java/io/nosqlbench/adapter/qdrant/ops) supported by the [Java
|
||||
driver published by Qdrant](https://github.com/qdrant/java-client). The official Qdrant API reference can be found at
|
||||
https://qdrant.github.io/java-client/io/qdrant/client/package-summary.html
|
||||
|
||||
The operations include a full-fledged support for all the APIs available in the Qdrant Java driver.
|
||||
The operations include a full-fledged support for key APIs available in the Qdrant Java driver.
|
||||
The following are a couple high level API operations.
|
||||
|
||||
# TODO - Below needs to be updated post driver development.
|
||||
* Create Collection
|
||||
* Create Index
|
||||
* Count Points
|
||||
* Drop Collection
|
||||
* Drop Index
|
||||
* Search (vectors)
|
||||
* Search Points (vectors)
|
||||
|
||||
## Examples
|
||||
```yaml
|
||||
ops:
|
||||
example_create_collection:
|
||||
create_collection: "example_collection"
|
||||
description: "https://qdrant.io/api-reference/java/v2.3.x/Collection/createCollection().md"
|
||||
collection_name: "example_collection"
|
||||
shards_num: 10
|
||||
consistency_level: BOUNDED # BOUNDED, SESSION, EVENTUAL
|
||||
field_types:
|
||||
field1:
|
||||
primary_key: true # only for Int64 and VarChar types
|
||||
description: "field description"
|
||||
data_type: "Varchar"
|
||||
# Bool, Int8, Int16, Int32, Int64,
|
||||
# Float, Double, String, Varchar, BinaryVector, FloatVector
|
||||
type_param:
|
||||
example_param1: example_pvalue1
|
||||
dimension: 1024 # >0
|
||||
max_length: 1024 # for String only, >0
|
||||
auto_id: false # Generate primary key?
|
||||
partition_key: false # Primary key cannot be the partition key too
|
||||
field2:
|
||||
primary_key: false
|
||||
description: "vector column/field"
|
||||
data_type: "FloatVector"
|
||||
dimension: 3
|
||||
|
||||
# https://qdrant.io/api-reference/java/v2.3.x/Index/dropIndex().md
|
||||
example_drop_index:
|
||||
drop_index: "exampe_collection_idx_name"
|
||||
database_name: "my_database"
|
||||
collection_name: "example_collection""
|
||||
Check out the [full example available here](activities/qdrant_vectors_live.yaml).
|
||||
|
||||
# https://qdrant.io/api-reference/java/v2.3.x/Collection/dropCollection().md
|
||||
example_drop_collection:
|
||||
drop_collection: "example_collection"
|
||||
database_name: "my_database"
|
||||
|
||||
# https://qdrant.io/api-reference/java/v2.3.x/High-level%20API/insert().md
|
||||
example_insert_op:
|
||||
insert: "example_collection_name"
|
||||
rows:
|
||||
field1: "row_key"
|
||||
field2: "[1.2, 3.4, 5.6]"
|
||||
|
||||
# https://qdrant.io/api-reference/java/v2.3.x/High-level%20API/search().md
|
||||
# https://qdrant.io/api-reference/java/v2.3.x/Query%20and%20Search/search().md
|
||||
example_search:
|
||||
search: "example_collection"
|
||||
vector: "[-0.4, 0.3, 0.99]"
|
||||
metric_type: "COSINE"
|
||||
out_fields:
|
||||
- field1
|
||||
- field2
|
||||
vector_field_name: "field2"
|
||||
top_k: 100
|
||||
consistency_level: "EVENTUALLY"
|
||||
```
|
||||
---
|
||||
|
||||
Loading…
Reference in New Issue
Block a user