IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-16 17:34:52 -06:00
Code Issues Packages Projects Releases Wiki Activity

cleanup dead code

Browse Source
This commit is contained in:
Madhavan Sridharan 2024-05-10 12:43:44 -04:00
parent 309fd6f7cc
commit e2534120ec
4 changed files with 6 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

267
nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java
View File

@ -26,7 +26,6 @@ import io.qdrant.client.grpc.Collections.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@ -58,20 +57,10 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
LongFunction<CreateCollection.Builder> ebF =
l -> CreateCollection.newBuilder().setCollectionName(targetF.apply(l));
// new code - incomplete
LongFunction<Map<String, VectorParams>> namedVectorParamsMap = buildNamedVectorsStruct(op);
final LongFunction<CreateCollection.Builder> namedVectorsF = ebF;
ebF = l -> namedVectorsF.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap.apply(l)).build()));
// new code - incomplete
// old code
// Map<String, VectorParams> namedVectorParamsMap1 = buildNamedVectorsStruct(
// op.getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey)
// );
// final LongFunction<CreateCollection.Builder> namedVectorsF1 = ebF;
// ebF = l -> namedVectorsF1.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap(
// VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap1).build()));
// old code
ebF = op.enhanceFuncOptionally(ebF, "on_disk_payload", Boolean.class,
CreateCollection.Builder::setOnDiskPayload);
@ -92,9 +81,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
LongFunction<WalConfigDiff> wcdF = buildWalConfigDiff(walF.get());
ebF = l -> wallFunc.apply(l).setWalConfig(wcdF.apply(l));
}
// WalConfigDiff walConfig = buildWalConfigDiff(op);
// final LongFunction<CreateCollection.Builder> walConfigF = ebF;
// ebF = l -> walConfigF.apply(l).setWalConfig(walConfig);
Optional<LongFunction<Map>> optConDifF = op.getAsOptionalFunction("optimizers_config", Map.class);
if (optConDifF.isPresent()) {
@ -103,10 +89,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
ebF = l -> wallFunc.apply(l).setOptimizersConfig(ocdF.apply(l));
}
// OptimizersConfigDiff ocDiff = buildOptimizerConfigDiff(op);
// final LongFunction<CreateCollection.Builder> ocF = ebF;
// ebF = l -> ocF.apply(l).setOptimizersConfig(ocDiff);
Optional<LongFunction<Map>> hnswConfigDiffF = op.getAsOptionalFunction("hnsw_config", Map.class);
if (hnswConfigDiffF.isPresent()) {
final LongFunction<CreateCollection.Builder> hnswConfigF = ebF;
@ -114,21 +96,12 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
ebF = l -> hnswConfigF.apply(l).setHnswConfig(hcdF.apply(l));
}
// HnswConfigDiff hnswConfigDiff = buildHnswConfigDiff(op);
// final LongFunction<CreateCollection.Builder> hnswConfigF = ebF;
// ebF = l -> hnswConfigF.apply(l).setHnswConfig(hnswConfigDiff);
Optional<LongFunction<Map>> quantConfigF = op.getAsOptionalFunction("quantization_config", Map.class);
if (quantConfigF.isPresent()) {
final LongFunction<CreateCollection.Builder> qConF = ebF;
LongFunction<QuantizationConfig> qcDiffF = buildQuantizationConfig(quantConfigF.get());
ebF = l -> qConF.apply(l).setQuantizationConfig(qcDiffF.apply(l));
}
// QuantizationConfig qcDiff = buildQuantizationConfig(op);
// if (qcDiff != null) {
// final LongFunction<CreateCollection.Builder> qcConfigF = ebF;
// ebF = l -> qcConfigF.apply(l).setQuantizationConfig(qcDiff);
// }
Optional<LongFunction<Map>> sparseVectorsF = op.getAsOptionalFunction("sparse_vectors", Map.class);
if (sparseVectorsF.isPresent()) {
@ -136,59 +109,11 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
LongFunction<SparseVectorConfig> sparseVectorsMap = buildSparseVectorsStruct(sparseVectorsF.get());
ebF = l -> sparseVecF.apply(l).setSparseVectorsConfig(sparseVectorsMap.apply(l));
}
// if (op.isDefined("sparse_vectors")) {
// SparseVectorConfig sparseVectorsMap = buildSparseVectorsStruct(
// op.getAsSubOps("sparse_vectors", ParsedOp.SubOpNaming.SubKey)
// );
// final LongFunction<CreateCollection.Builder> sparseVectorsF = ebF;
// ebF = l -> sparseVectorsF.apply(l).setSparseVectorsConfig(sparseVectorsMap);
// }
final LongFunction<CreateCollection.Builder> lastF = ebF;
return l -> lastF.apply(l).build();
}
/**
* Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}.
*
* @param op {@link ParsedOp} containing the optimizer config data.
* @return {@link OptimizersConfigDiff} containing the optimizer config data
*/
private OptimizersConfigDiff buildOptimizerConfigDiff(ParsedOp op) {
OptimizersConfigDiff.Builder ocDiffBuilder = OptimizersConfigDiff.newBuilder();
op.getOptionalStaticValue("optimizers_config", Map.class).ifPresent(ocData -> {
if (ocData.isEmpty()) {
return;
} else {
if (ocData.containsKey("deleted_threshold")) {
ocDiffBuilder.setDeletedThreshold(((Number) ocData.get("deleted_threshold")).doubleValue());
}
if (ocData.containsKey("vacuum_min_vector_number")) {
ocDiffBuilder.setVacuumMinVectorNumber(((Number) ocData.get("vacuum_min_vector_number")).longValue());
}
if (ocData.containsKey("default_segment_number")) {
ocDiffBuilder.setDefaultSegmentNumber(((Number) ocData.get("default_segment_number")).longValue());
}
if (ocData.containsKey("max_segment_size")) {
ocDiffBuilder.setMaxSegmentSize(((Number) ocData.get("max_segment_size")).longValue());
}
if (ocData.containsKey("memmap_threshold")) {
ocDiffBuilder.setMemmapThreshold(((Number) ocData.get("memmap_threshold")).longValue());
}
if (ocData.containsKey("indexing_threshold")) {
ocDiffBuilder.setIndexingThreshold(((Number) ocData.get("indexing_threshold")).longValue());
}
if (ocData.containsKey(("flush_interval_sec"))) {
ocDiffBuilder.setFlushIntervalSec(((Number) ocData.get("flush_interval_sec")).longValue());
}
if (ocData.containsKey("max_optimization_threads")) {
ocDiffBuilder.setMaxOptimizationThreads(((Number) ocData.get("max_optimization_threads")).intValue());
}
}
});
return ocDiffBuilder.build();
}
/**
* Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}.
*
@ -229,30 +154,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
};
}
/**
* Build the {@link WalConfigDiff} from the provided {@link ParsedOp}.
*
* @param op {@link ParsedOp} containing the WAL config data.
* @return {@link WalConfigDiff} containing the WAL config data
*/
@Deprecated
private WalConfigDiff buildWalConfigDiff(ParsedOp op) {
WalConfigDiff.Builder walConfigDiffBuilder = WalConfigDiff.newBuilder();
op.getOptionalStaticValue("wal_config", Map.class).ifPresent(walConfigData -> {
if (walConfigData.isEmpty()) {
return;
} else {
if (walConfigData.containsKey("wal_capacity_mb")) {
walConfigDiffBuilder.setWalCapacityMb(((Number) walConfigData.get("wal_capacity_mb")).longValue());
}
if (walConfigData.containsKey("wal_segments_ahead")) {
walConfigDiffBuilder.setWalSegmentsAhead(((Number) walConfigData.get("wal_segments_ahead")).longValue());
}
}
});
return walConfigDiffBuilder.build();
}
/**
* Build the {@link WalConfigDiff} from the provided {@link ParsedOp}.
*
@ -275,32 +176,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
};
}
/**
* Only named vectors are supported at this time in this driver.
*
* @param {@link Map<String, ParsedOp>} namedVectorsData
* @return {@link VectorParams} containing the named vectors
*/
@Deprecated
private Map<String, VectorParams> buildNamedVectorsStruct(Map<String, ParsedOp> namedVectorsData) {
Map<String, VectorParams> namedVectors = new HashMap<>();
VectorParams.Builder builder = VectorParams.newBuilder();
namedVectorsData.forEach((name, fieldSpec) -> {
builder.setDistanceValue(fieldSpec.getStaticValue("distance_value", Number.class).intValue());
builder.setSize(fieldSpec.getStaticValue("size", Number.class).longValue());
fieldSpec.getOptionalStaticValue("on_disk", Boolean.class)
.ifPresent(builder::setOnDisk);
fieldSpec.getOptionalStaticValue("datatype_value", Number.class)
.ifPresent((Number value) -> builder.setDatatypeValue(value.intValue()));
builder.setHnswConfig(buildHnswConfigDiff(fieldSpec));
builder.setQuantizationConfig(buildQuantizationConfig(fieldSpec));
namedVectors.put(name, builder.build());
});
return namedVectors;
}
/**
* Only named vectors are supported at this time in this driver.
*
@ -348,124 +223,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
}).orElse(null);
}
/**
* Build the {@link QuantizationConfig} from the provided {@link ParsedOp}.
*
* @param fieldSpec The {@link ParsedOp} containing the quantization config data
* @return The {@link QuantizationConfig} built from the provided {@link ParsedOp}
* @see <a href="https://qdrant.tech/documentation/guides/quantization/#setting-up-quantization-in-qdrant">Quantization Config</a>
*/
@Deprecated
private QuantizationConfig buildQuantizationConfig(ParsedOp fieldSpec) {
QuantizationConfig.Builder qcBuilder = QuantizationConfig.newBuilder();
fieldSpec.getOptionalStaticValue("quantization_config", Map.class).ifPresent(qcData -> {
if (qcData.isEmpty()) {
return;
} else {
// TODO - Approach #1 - feels ugly
Arrays.asList("binary", "product", "scalar")
.forEach(key -> {
if (qcData.containsKey(key)) {
switch (key) {
case "binary":
BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder();
Map<?, ?> binaryQCData = (Map<?, ?>) qcData.get("binary");
if (null != binaryQCData && !binaryQCData.isEmpty()) {
if (binaryQCData.containsKey("always_ram")) {
binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram"));
}
}
qcBuilder.setBinary(binaryBuilder);
break;
case "product":
ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder();
Map<?, ?> productQCData = (Map<?, ?>) qcData.get("product");
if (null != productQCData && !productQCData.isEmpty()) {
// Mandatory field
productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram"));
// Optional field(s) below
if (productQCData.containsKey("compression")) {
productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression")));
}
}
qcBuilder.setProduct(productBuilder);
break;
case "scalar":
ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder();
Map<?, ?> scalarQCData = (Map<?, ?>) qcData.get("scalar");
if (null != scalarQCData && !scalarQCData.isEmpty()) {
// Mandatory field
scalarBuilder.setType(QuantizationType.forNumber(((Number) scalarQCData.get("type")).intValue()));
// Optional field(s) below
if (scalarQCData.containsKey("always_ram")) {
scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram"));
}
if (scalarQCData.containsKey("quantile")) {
scalarBuilder.setQuantile(((Number) scalarQCData.get("quantile")).floatValue());
}
}
qcBuilder.setScalar(scalarBuilder);
break;
}
}
});
// TODO - Approach #2 - equally feels ugly too.
// if (qcData.containsKey("binary")) {
// if (qcData.containsKey("scalar") || qcData.containsKey("product")) {
// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config");
// }
// BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder();
// Map<?, ?> binaryQCData = (Map<?, ?>) qcData.get("binary");
// if (null != binaryQCData && !binaryQCData.isEmpty()) {
// if (binaryQCData.containsKey("always_ram")) {
// binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram"));
// }
// }
// qcBuilder.setBinary(binaryBuilder);
// } else if (qcData.containsKey("product")) {
// if (qcData.containsKey("binary") || qcData.containsKey("scalar")) {
// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config");
// }
// ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder();
// Map<?, ?> productQCData = (Map<?, ?>) qcData.get("product");
// if (null != productQCData && !productQCData.isEmpty()) {
// // Mandatory field
// productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram"));
// // Optional field(s) below
// if (productQCData.containsKey("compression")) {
// productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression")));
// }
// }
// qcBuilder.setProduct(productBuilder);
// } else if (qcData.containsKey("scalar")) {
// if (qcData.containsKey("binary") || qcData.containsKey("product")) {
// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config");
// }
// ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder();
// Map<?, ?> scalarQCData = (Map<?, ?>) qcData.get("scalar");
// if (null != scalarQCData && !scalarQCData.isEmpty()) {
// // Mandatory field
// scalarBuilder.setType(QuantizationType.valueOf((String) scalarQCData.get("type")));
// // Optional field(s) below
// if (scalarQCData.containsKey("always_ram")) {
// scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram"));
// }
// if (scalarQCData.containsKey("quantile")) {
// scalarBuilder.setQuantile((Float) scalarQCData.get("quantile"));
// }
// }
// qcBuilder.setScalar(scalarBuilder);
// }
}
});
// The below check is required to avoid INVALID_ARGUMENT: Unable to convert quantization config
if (qcBuilder.hasBinary() || qcBuilder.hasProduct() || qcBuilder.hasScalar()) {
return qcBuilder.build();
}
return null;
}
private LongFunction<QuantizationConfig> buildQuantizationConfig(LongFunction<Map> quantConfMapLongFunc) {
return l -> this.buildQuantizationConfig(quantConfMapLongFunc.apply(l));
}
@ -592,30 +349,6 @@ public class QdrantCreateCollectionOpDispenser extends QdrantBaseOpDispenser<Cre
return hnswConfigBuilder.build();
}
/**
* Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}.
*
* @param sparseVectorsData The {@link ParsedOp} containing the sparse vectors data
* @return The {@link SparseVectorConfig} built from the provided {@link ParsedOp}
*/
@Deprecated
private SparseVectorConfig buildSparseVectorsStruct(Map<String, ParsedOp> sparseVectorsData) {
SparseVectorConfig.Builder builder = SparseVectorConfig.newBuilder();
sparseVectorsData.forEach((name, fieldSpec) -> {
SparseVectorParams.Builder svpBuilder = SparseVectorParams.newBuilder();
SparseIndexConfig.Builder sicBuilder = SparseIndexConfig.newBuilder();
fieldSpec.getOptionalStaticValue("full_scan_threshold", Number.class)
.ifPresent((Number value) -> sicBuilder.setFullScanThreshold(value.intValue()));
fieldSpec.getOptionalStaticValue("on_disk", Boolean.class)
.ifPresent(sicBuilder::setOnDisk);
svpBuilder.setIndex(sicBuilder);
builder.putMap(name, svpBuilder.build());
});
return builder.build();
}
/**
* Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}.
*

25
nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java
View File

@ -216,29 +216,4 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser<SearchP
}
};
}
@Deprecated
private Optional<LongFunction<List<Float>>> getVectorFieldsFunction(ParsedOp op, String namedVectors) {
// return l -> {
// if (!op.isDefined(namedVectors)) {
// return Optional.empty();
// }
// List<Float> fields = op.get(namedVectors, 0L);
// if (fields == null) {
// fields = op.get(namedVectors, 0L);
// }
// return fields;
// };
LongFunction<Object> rowF = op.getAsRequiredFunction(namedVectors, Object.class);
Object testObject = rowF.apply(0L);
LongFunction<List<Float>> rowsF = null;
if(testObject instanceof List<?> list) {
if (list.isEmpty()) {
throw new OpConfigError("Unable to detect type of list object for empty list for op named '" + op.getName() + "'");
} else if (list.get(0) instanceof Float) {
rowsF = l -> (List<Float>) rowF.apply(l);
}
}
return Optional.ofNullable(rowsF);
}
}

2
nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java
View File

@ -37,7 +37,7 @@ public class QdrantCountPointsOp extends QdrantBaseOp<CountPoints> {
request.getExact(),
Duration.ofMinutes(5) // opinionated default of 5 minutes for timeout
).get();
logger.info("Total vector points counted: {}", result);
logger.info("[QdrantCountPointsOp] Total vector points counted: {}", result);
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}

10
nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml
View File

@ -17,27 +17,27 @@ scenarios:
run tags==block:delete_collection
errors===stop
cycles===UNDEF threads===UNDEF
uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file)
uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file)
schema_collection: >-
run tags==block:schema_collection
errors===stop
cycles===UNDEF threads===UNDEF
uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file)
uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file)
rampup: >-
run tags==block:rampup
errors===warn,counter,retry
cycles===TEMPLATE(train_cycles,TEMPLATE(trainsize,1000)) threads===TEMPLATE(train_threads,AUTO)
uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file)
uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file)
count_vectors: >-
run tags==block:count_vectors
errors===stop
cycles===UNDEF threads===UNDEF
uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file)
uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file)
search_points: >-
run tags==block:search_points
errors===warn,counter
cycles===TEMPLATE(testann_cycles,TEMPLATE(testsize,1000)) threads===TEMPLATE(testann_threads,AUTO)
uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file)
uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file)
params:
driver: qdrant