IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

make m,ef_construction,dimensions static in os index create

Browse Source
This commit is contained in:
Jonathan Shook 2024-02-14 18:27:30 -06:00
parent 4ad42f8882
commit 33389ea4f3
2 changed files with 17 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/CreateIndexOpDispenser.java
View File

@ -30,8 +30,17 @@ import java.util.function.LongFunction;
public class CreateIndexOpDispenser extends BaseOpenSearchOpDispenser {
private final ParsedOp pop;
private final int dimensions;
private final int ef_construction;
private final int m;
public CreateIndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op, LongFunction<String> targetF) {
super(adapter, op, targetF);
this.pop = op;
this.dimensions = pop.getStaticValue("dimensions",Integer.class).intValue();
this.ef_construction = pop.getStaticValue("ef_construction",Integer.class).intValue();
this.m = pop.getStaticValue("m",Integer.class).intValue();
}
@Override
@ -50,6 +59,7 @@ public class CreateIndexOpDispenser extends BaseOpenSearchOpDispenser {
// https://opensearch.org/docs/latest/search-plugins/knn/knn-index/
private CreateIndexRequest.Builder resolveTypeMapping(CreateIndexRequest.Builder eb, Map<?, ?> mappings) {
TypeMapping.Builder builder = new TypeMapping.Builder().properties(
Map.of(
"key",
@ -59,13 +69,18 @@ public class CreateIndexOpDispenser extends BaseOpenSearchOpDispenser {
"value",
new Property.Builder()
.knnVector(new KnnVectorProperty.Builder()
.dimension(25)
.dimension(dimensions)
.method(
new KnnVectorMethod.Builder()
.name("hnsw")
.engine("faiss")
.spaceType("l2")
.parameters(Map.of("ef_construction", JsonData.of(256), "m", JsonData.of(8)))
.parameters(
Map.of(
"ef_construction", JsonData.of(ef_construction),
"m", JsonData.of(m)
)
)
.build()
).build()
).build()

112
adapter-aws-opensearch/src/main/resources/activities/osvectors.yaml
View File

@ -1,112 +0,0 @@
description: |
basic test for vectors in open search
https://www.elastic.co/guide/en/elasticsearch/reference/current/rest-apis.html
template vars:
TEMPLATE(indexname,vectors_index)
TEMPLATE(dimensions,25)
TEMPLATE(search_cycles,1M)
TEMPLATE(rampup_cycles,TEMPLATE(trainsize))
TEMPLATE(size,100)
params:
driver: opensearch
instrument: true
scenarios:
vectors_brief:
bulkrampup: >-
run tags='block:bulkrampup' labels='target:opensearch'
threads=TEMPLATE(rampup_threads,10) cycles=TEMPLATE(trainsize)
errors=count,warn
# search: run tags='block:search' labels='target:opensearch' threads=TEMPLATE(search_threads,10) cycles=TEMPLATE(testsize)
#rampup: >-
# run tags='block:rampup' labels='target:opensearch'
# threads=TEMPLATE(rampup_threads,10) cycles=TEMPLATE(trainsize)
# errors=count,warn
## search: run tags='block:search' labels='target:opensearch' threads=TEMPLATE(search_threads,10) cycles=TEMPLATE(testsize)
search_and_verify: >-
run tags='block:search_and_verify' labels='target:opensearch'
threads=TEMPLATE(search_threads,10) cycles=TEMPLATE(testsize)
errors=count,warn
vectors:
# drop: run tags='block:drop' labels='target:opensearch' threads===1 cycles===UNDEF
schema: run tags='block:schema' labels='target:opensearch' threads===1 cycles===UNDEF
rampup: >-
run tags='block:rampup' labels='target:opensearch'
threads=TEMPLATE(rampup_threads,10) cycles=TEMPLATE(trainsize)
errors=count,warn
# search: run tags='block:search' labels='target:opensearch' threads=TEMPLATE(search_threads,10) cycles=TEMPLATE(testsize)
search_and_verify: >-
run tags='block:search_and_verify' labels='target:opensearch'
threads=TEMPLATE(search_threads,10) cycles=TEMPLATE(testsize)
errors=count,warn
# errors=counter,warn,log
bindings:
id: ToString()
test_floatlist: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/test");
relevant_indices: HdfFileToIntArray("testdata/TEMPLATE(dataset).hdf5", "/neighbors")
distance_floatlist: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/distance")
train_floatlist: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/train");
blocks:
drop:
ops:
drop_index:
delete_index: TEMPLATE(indexname,vectors_index)
schema:
ops:
create_index:
create_index: TEMPLATE(indexname)
mappings:
m1: v1
search:
ops:
search:
knn_search: TEMPLATE(indexname,vectors_index)
k: 100
vector: "{test_floatlist}"
field: value
search_and_verify:
ops:
select_ann_limit_TEMPLATE(k,100):
knn_search: TEMPLATE(indexname,vectors_index)
k: 100
vector: "{test_floatlist}"
field: value
verifier-init: |
relevancy=new io.nosqlbench.nb.api.engine.metrics.wrappers.RelevancyMeasures(_parsed_op)
for (int k in List.of(100)) {
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("recall",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("precision",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("F1",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("RR",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("AP",k));
}
verifier: |
// driver-specific function
actual_indices=io.nosqlbench.adapter.opensearch.Utils.DocHitsToIntIndicesArray(result)
// driver-agnostic function
relevancy.accept({relevant_indices},actual_indices);
return true;
bulkrampup:
ops:
bulk_index:
bulk: TEMPLATE(indexname)
op_template:
repeat: TEMPLATE(bulk_repeat,100)
index: TEMPLATE(indexname)
document:
key: "{id}"
value: "{train_floatlist}"
rampup:
ops:
index:
index: TEMPLATE(indexname)
document:
key: "{id}"
value: "{train_floatlist}"