mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-01-11 08:22:04 -06:00
consolidate
This commit is contained in:
Madhavan Sridharan
parent
2f6be23263
commit
011f477afb
@ -16,6 +16,9 @@
|
||||
|
||||
package io.nosqlbench.adapter.milvus;
|
||||
|
||||
import io.milvus.grpc.SearchResults;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.response.SearchResultsWrapper;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Arrays;
|
||||
@ -59,4 +62,14 @@ public class MilvusAdapterUtils {
|
||||
}
|
||||
return masked.toString();
|
||||
}
|
||||
|
||||
public static int[] intArrayFromMilvusSearchResults(String fieldName, R<SearchResults> result) {
|
||||
SearchResultsWrapper wrapper = new SearchResultsWrapper(result.getData().getResults());
|
||||
List<String> fieldData = (List<String>) wrapper.getFieldData(fieldName, 0);
|
||||
int[] indices = new int[fieldData.size()];
|
||||
for (int i = 0; i < indices.length; i++) {
|
||||
indices[i] = Integer.parseInt(fieldData.get(i));
|
||||
}
|
||||
return indices;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2024 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.adapter.milvus;
|
||||
|
||||
import io.milvus.grpc.SearchResults;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.response.SearchResultsWrapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class MilvusUtils {
|
||||
|
||||
public static int[] intArrayFromMilvusSearchResults(String fieldname, R<SearchResults> result) {
|
||||
|
||||
SearchResultsWrapper wrapper = new SearchResultsWrapper(result.getData().getResults());
|
||||
List<String> fieldData = (List<String>) wrapper.getFieldData(fieldname, 0);
|
||||
int[] indices = new int[fieldData.size()];
|
||||
for (int i = 0; i < indices.length; i++) {
|
||||
indices[i]=Integer.parseInt(fieldData.get(i));
|
||||
}
|
||||
return indices;
|
||||
|
||||
}
|
||||
}
|
||||
@ -19,8 +19,8 @@ package io.nosqlbench.adapter.milvus.opdispensers;
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.common.clientenum.ConsistencyLevelEnum;
|
||||
import io.milvus.param.highlevel.dml.GetIdsParam;
|
||||
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
|
||||
import io.nosqlbench.adapter.milvus.MilvusAdapterUtils;
|
||||
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
|
||||
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
|
||||
import io.nosqlbench.adapter.milvus.ops.MilvusGetOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
@ -59,7 +59,7 @@ public class MilvusGetOpDispenser extends MilvusBaseOpDispenser<GetIdsParam> {
|
||||
LongFunction<GetIdsParam.Builder> ebF =
|
||||
l -> GetIdsParam.newBuilder();
|
||||
|
||||
// ebF = l -> ebF.apply(l).withPrimaryIds(MilvusUtils.splitNames(targetF.apply(l)));
|
||||
// ebF = l -> ebF.apply(l).withPrimaryIds(MilvusAdapterUtils.splitNames(targetF.apply(l)));
|
||||
LongFunction<Object> valueFunc = op.getAsRequiredFunction("primary_ids", Object.class);
|
||||
Object testValue = valueFunc.apply(0L);
|
||||
LongFunction<List<Object>> pidsF;
|
||||
@ -75,7 +75,7 @@ public class MilvusGetOpDispenser extends MilvusBaseOpDispenser<GetIdsParam> {
|
||||
LongFunction<GetIdsParam.Builder> finalEbF2 = ebF;
|
||||
ebF = l -> finalEbF2.apply(l).withPrimaryIds(pidsF.apply(l));
|
||||
|
||||
ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name","collection"), String.class,
|
||||
ebF = op.enhanceFuncOptionally(ebF, List.of("collection_name", "collection"), String.class,
|
||||
GetIdsParam.Builder::withCollectionName);
|
||||
ebF = op.enhanceEnumOptionally(ebF, "consistency_level", ConsistencyLevelEnum.class,
|
||||
GetIdsParam.Builder::withConsistencyLevel);
|
||||
|
||||
@ -42,7 +42,7 @@ public class MilvusLoadBalanceOpDispenser extends MilvusBaseOpDispenser<LoadBala
|
||||
LongFunction<LoadBalanceParam.Builder> ebF =
|
||||
l -> LoadBalanceParam.newBuilder();
|
||||
LongFunction<String> segmentsStringF = op.getAsRequiredFunction("segment_ids");
|
||||
// ebF = l -> ebF.apply(l).withSegmentIDs(MilvusUtils.splitLongs(segmentsStringF.apply(l)));
|
||||
// ebF = l -> ebF.apply(l).withSegmentIDs(MilvusAdapterUtils.splitLongs(segmentsStringF.apply(l)));
|
||||
|
||||
throw new RuntimeException("implement me");
|
||||
// todo: implement me
|
||||
@ -59,6 +59,6 @@ public class MilvusLoadBalanceOpDispenser extends MilvusBaseOpDispenser<LoadBala
|
||||
ParsedOp op,
|
||||
LongFunction<String> targetF
|
||||
) {
|
||||
return l -> new MilvusLoadBalanceOp(clientF.apply(l),paramF.apply(l));
|
||||
return l -> new MilvusLoadBalanceOp(clientF.apply(l), paramF.apply(l));
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,10 +63,10 @@ blocks:
|
||||
drop_index_op:
|
||||
drop_index: "TEMPLATE(collection)_TEMPLATE(vector_field,value)_idx"
|
||||
collection_name: "TEMPLATE(collection)"
|
||||
# database_name: "TEMPLATE(database)"
|
||||
# database_name: "TEMPLATE(database)"
|
||||
drop_col_op:
|
||||
drop_collection: "TEMPLATE(collection)"
|
||||
# database_name: "TEMPLATE(database)"
|
||||
# database_name: "TEMPLATE(database)"
|
||||
# https://milvus.io/api-reference/java/v2.3.x/Index/dropIndex().md
|
||||
drop_db_op:
|
||||
drop_database: "TEMPLATE(database)"
|
||||
@ -86,7 +86,7 @@ blocks:
|
||||
data_type: "VarChar"
|
||||
max_length: 1024
|
||||
auto_id: false
|
||||
# partition_key: true
|
||||
# partition_key: true
|
||||
value:
|
||||
primary_key: false
|
||||
description: "vector column/field"
|
||||
@ -110,9 +110,9 @@ blocks:
|
||||
rows:
|
||||
key: "{row_key}"
|
||||
value: "{train_floatlist_TEMPLATE(filetype)}"
|
||||
# fields:
|
||||
# key: (List this) "{row_key}"
|
||||
# value: (List this) "{train_floatlist_TEMPLATE(filetype)}"
|
||||
# fields:
|
||||
# key: (List this) "{row_key}"
|
||||
# value: (List this) "{train_floatlist_TEMPLATE(filetype)}"
|
||||
|
||||
search:
|
||||
ops:
|
||||
@ -137,16 +137,16 @@ blocks:
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("RR",k));
|
||||
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("AP",k));
|
||||
}
|
||||
# for (int k in List.of(1,2,3,5,10,25,50,75)) {
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("s_recall",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("s_precision",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("s_F1",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("s_RR",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("s_AP",k));
|
||||
# }
|
||||
# for (int k in List.of(1,2,3,5,10,25,50,75)) {
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("s_recall",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("s_precision",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("s_F1",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("s_RR",k));
|
||||
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("s_AP",k));
|
||||
# }
|
||||
verifier: |
|
||||
// driver-specific function
|
||||
actual_indices=io.nosqlbench.adapter.milvus.MilvusUtils.intArrayFromMilvusSearchResults("key",result)
|
||||
actual_indices=io.nosqlbench.adapter.milvus.MilvusAdapterUtils.intArrayFromMilvusSearchResults("key",result)
|
||||
// driver-agnostic function
|
||||
relevancy.accept({relevant_indices_TEMPLATE(filetype)},actual_indices);
|
||||
// because we are "verifying" although this needs to be reorganized
|
||||
|
||||
Loading…
Reference in New Issue
Block a user