IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'adapter/milvus' of github.com:nosqlbench/nosqlbench into adapter/milvus

Browse Source
This commit is contained in:
Madhavan Sridharan
2024-03-21 17:01:19 -04:00
parent 16c17e9975 51a7dd3cdc
commit 2f6be23263
22 changed files with 275 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.run/milvus drop localhost.run.xml Normal file
View File

@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="milvus drop localhost" type="JarApplication" folderName="Milvus">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="milvus default.drop milvushost=localhost --show-stacktraces --progress console:1s -v" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/milvus" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>

14
.run/milvus drop on aws.run.xml Normal file
View File

@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="milvus drop on aws" type="JarApplication" folderName="Milvus">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="milvus default.drop milvushost=milvushost --show-stacktraces --progress console:1s -v" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/milvus" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>

14
.run/milvus rampup localhost smoketest .run.xml Normal file
View File

@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="milvus rampup localhost smoketest " type="JarApplication" folderName="Milvus">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="milvus default.rampup rampup_threads=1 milvushost=localhost datafile=&quot;vector/ANN/glove-25-angular/glove-25-angular&quot; --show-stacktraces --progress console:1s -v" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/milvus" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>

14
.run/milvus schema localhost.run.xml Normal file
View File

@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="milvus schema localhost" type="JarApplication" folderName="Milvus">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="milvus default.schema milvushost=localhost --show-stacktraces --progress console:1s -v" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/milvus" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>

14
.run/milvus schema on aws.run.xml Normal file
View File

@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="milvus schema on aws" type="JarApplication" folderName="Milvus">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="milvus default.schema milvushost=milvushost --show-stacktraces --progress console:1s -v" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/milvus" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>

62
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusAdapterUtils.java Normal file
View File

@@ -0,0 +1,62 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.milvus;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.List;
public class MilvusAdapterUtils {
public static final String MILVUS = "milvus";
public static List<String> splitNames(String input) {
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
return Arrays.stream(input.split("( +| *, *)"))
.filter(StringUtils::isNotBlank)
.toList();
}
public static List<Long> splitLongs(String input) {
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
return Arrays.stream(input.split("( +| *, *)"))
.filter(StringUtils::isNotBlank)
.map(Long::parseLong)
.toList();
}
/**
* Mask the digits in the given string with '*'
*
* @param unmasked The string to mask
* @return The masked string
*/
protected static String maskDigits(String unmasked) {
assert StringUtils.isNotBlank(unmasked) && StringUtils.isNotEmpty(unmasked);
int inputLength = unmasked.length();
StringBuilder masked = new StringBuilder(inputLength);
for (char ch : unmasked.toCharArray()) {
if (Character.isDigit(ch)) {
masked.append("*");
} else {
masked.append(ch);
}
}
return masked.toString();
}
}

2
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusDriverAdapter.java
View File

@@ -28,7 +28,7 @@ import io.nosqlbench.nb.api.labels.NBLabels;
import java.util.function.Function;
import static io.nosqlbench.adapter.milvus.MilvusUtils.MILVUS;
import static io.nosqlbench.adapter.milvus.MilvusAdapterUtils.MILVUS;
@Service(value = DriverAdapter.class, selector = MILVUS)
public class MilvusDriverAdapter extends BaseDriverAdapter<MilvusBaseOp<?>, MilvusSpace> {

2
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusDriverAdapterLoader.java
View File

@@ -21,7 +21,7 @@ import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.labels.NBLabels;
import static io.nosqlbench.adapter.milvus.MilvusUtils.MILVUS;
import static io.nosqlbench.adapter.milvus.MilvusAdapterUtils.MILVUS;
@Service(value = DriverAdapterLoader.class, selector = MILVUS)
public class MilvusDriverAdapterLoader implements DriverAdapterLoader {

8
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java
View File

@@ -27,11 +27,10 @@ import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* The MilvusSpace class is a context object which stores all stateful contextual information needed to interact
@@ -73,6 +72,7 @@ public class MilvusSpace implements AutoCloseable {
var builder = ConnectParam.newBuilder();
builder = builder.withUri(cfg.get("uri"));
cfg.getOptional("database_name").ifPresent(builder::withDatabaseName);
cfg.getOptional("database").ifPresent(builder::withDatabaseName);
var requiredToken = cfg.getOptional("token_file")
.map(Paths::get)
@@ -97,7 +97,7 @@ public class MilvusSpace implements AutoCloseable {
ConnectParam connectParams = builder.build();
logger.info(this.name + ": Creating new Milvus/Zilliz Client with (masked) " +
"token [" + MilvusUtils.maskDigits(builder.getToken()) + "], uri/endpoint [" + builder.getUri() + "]"
"token [" + MilvusAdapterUtils.maskDigits(builder.getToken()) + "], uri/endpoint [" + builder.getUri() + "]"
);
return new MilvusServiceClient(connectParams);
}
@@ -117,7 +117,7 @@ public class MilvusSpace implements AutoCloseable {
.setDescription("the URI endpoint in which the database is running.")
)
.add(
Param.optional("database_name")
Param.optional(List.of("database_name","database"))
.setDescription("the name of the database to use. Defaults to 'baselines'")
)
.asReadOnly();

46
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusUtils.java
View File

@@ -16,47 +16,23 @@
package io.nosqlbench.adapter.milvus;
import org.apache.commons.lang3.StringUtils;
import io.milvus.grpc.SearchResults;
import io.milvus.param.R;
import io.milvus.response.SearchResultsWrapper;
import java.util.Arrays;
import java.util.List;
public class MilvusUtils {
public static final String MILVUS = "milvus";
public static List<String> splitNames(String input) {
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
return Arrays.stream(input.split("( +| *, *)"))
.filter(StringUtils::isNotBlank)
.toList();
}
public static int[] intArrayFromMilvusSearchResults(String fieldname, R<SearchResults> result) {
public static List<Long> splitLongs(String input) {
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
return Arrays.stream(input.split("( +| *, *)"))
.filter(StringUtils::isNotBlank)
.map(Long::parseLong)
.toList();
}
/**
* Mask the digits in the given string with '*'
*
* @param unmasked The string to mask
* @return The masked string
*/
protected static String maskDigits(String unmasked) {
assert StringUtils.isNotBlank(unmasked) && StringUtils.isNotEmpty(unmasked);
int inputLength = unmasked.length();
StringBuilder masked = new StringBuilder(inputLength);
for (char ch : unmasked.toCharArray()) {
if (Character.isDigit(ch)) {
masked.append("*");
} else {
masked.append(ch);
}
SearchResultsWrapper wrapper = new SearchResultsWrapper(result.getData().getResults());
List<String> fieldData = (List<String>) wrapper.getFieldData(fieldname, 0);
int[] indices = new int[fieldData.size()];
for (int i = 0; i < indices.length; i++) {
indices[i]=Integer.parseInt(fieldData.get(i));
}
return masked.toString();
return indices;
}
}

5
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadStateOpDispenser.java
View File

@@ -19,12 +19,11 @@ package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.collection.GetLoadStateParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.MilvusAdapterUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusGetLoadStateOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.function.LongFunction;
@@ -52,7 +51,7 @@ public class MilvusGetLoadStateOpDispenser extends MilvusBaseOpDispenser<GetLoad
if (partitionsF.isPresent()) {
LongFunction<String> pfunc = partitionsF.get();
LongFunction<GetLoadStateParam.Builder> finalEbF = ebF;
ebF = l -> finalEbF.apply(l).withPartitionNames(MilvusUtils.splitNames(pfunc.apply(l)));
ebF = l -> finalEbF.apply(l).withPartitionNames(MilvusAdapterUtils.splitNames(pfunc.apply(l)));
}
final LongFunction<GetLoadStateParam.Builder> lastF = ebF;

4
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetLoadingProgressOpDispenser.java
View File

@@ -18,16 +18,12 @@ package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.collection.GetLoadingProgressParam;
import io.milvus.param.collection.GetLoadingProgressParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusGetLoadStateOp;
import io.nosqlbench.adapter.milvus.ops.MilvusGetLoadingProgressOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import java.util.List;
import java.util.Optional;
import java.util.function.LongFunction;
public class MilvusGetLoadingProgressOpDispenser extends MilvusBaseOpDispenser<GetLoadingProgressParam> {

8
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusGetOpDispenser.java
View File

@@ -20,7 +20,7 @@ import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.param.highlevel.dml.GetIdsParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.MilvusAdapterUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusGetOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
@@ -28,8 +28,6 @@ import io.nosqlbench.adapters.api.templating.ParsedOp;
import java.util.List;
import java.util.function.LongFunction;
import static org.matheclipse.core.expression.S.l;
/**
* Because we are using type-and-target logic to identify the op variant,
* we are limited to a string target value. In this particular op type, the
@@ -66,7 +64,7 @@ public class MilvusGetOpDispenser extends MilvusBaseOpDispenser<GetIdsParam> {
Object testValue = valueFunc.apply(0L);
LongFunction<List<Object>> pidsF;
if (testValue instanceof String string) {
pidsF = l -> MilvusUtils.splitNames((String) valueFunc.apply(l))
pidsF = l -> MilvusAdapterUtils.splitNames((String) valueFunc.apply(l))
.stream().map(s -> (Object) s).toList();
} else if (testValue instanceof List) {
pidsF = l -> (List<Object>) valueFunc.apply(l);
@@ -95,7 +93,7 @@ public class MilvusGetOpDispenser extends MilvusBaseOpDispenser<GetIdsParam> {
} else if (oftv instanceof String) {
var sF = op.getAsRequiredFunction("output_fields", String.class);
LongFunction<GetIdsParam.Builder> finalEbF1 = ebF;
ebF = l -> finalEbF1.apply(l).withOutputFields(MilvusUtils.splitNames(sF.apply(l)));
ebF = l -> finalEbF1.apply(l).withOutputFields(MilvusAdapterUtils.splitNames(sF.apply(l)));
} else throw new RuntimeException("Invalid type for output fields:" + oftv.getClass().getCanonicalName());
}

1
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusLoadBalanceOpDispenser.java
View File

@@ -19,7 +19,6 @@ package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.control.LoadBalanceParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusLoadBalanceOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;

4
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusReleasePartitionsOpDispenser.java
View File

@@ -19,7 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.partition.ReleasePartitionsParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.MilvusAdapterUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusReleasePartitionsOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
@@ -43,7 +43,7 @@ public class MilvusReleasePartitionsOpDispenser extends MilvusBaseOpDispenser<Re
) {
LongFunction<ReleasePartitionsParam.Builder> ebF =
l -> ReleasePartitionsParam.newBuilder();
LongFunction<List<String>> partNamesF = l -> MilvusUtils.splitNames(targetF.apply(l));
LongFunction<List<String>> partNamesF = l -> MilvusAdapterUtils.splitNames(targetF.apply(l));
LongFunction<ReleasePartitionsParam.Builder> finalEbF = ebF;
ebF = l -> finalEbF.apply(l).withPartitionNames(partNamesF.apply(l));

14
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusSearchOpDispenser.java
View File

@@ -47,9 +47,8 @@ public class MilvusSearchOpDispenser extends MilvusBaseOpDispenser<SearchParam>
LongFunction<SearchParam.Builder> ebF =
l -> SearchParam.newBuilder().withCollectionName(targetF.apply(l));
ebF = op.enhanceFuncOptionally(ebF,List.of("partition_names","partitions"),List.class,
SearchParam.Builder::withPartitionNames);
ebF = op.enhanceFuncOptionally(ebF,"out_fields",List.class,SearchParam.Builder::withOutFields);
ebF = op.enhanceFuncOptionally(ebF, List.of("partition_names", "partitions"), List.class, SearchParam.Builder::withPartitionNames);
ebF = op.enhanceFuncOptionally(ebF, "out_fields", List.class, SearchParam.Builder::withOutFields);
ebF = op.enhanceEnumOptionally(ebF, "consistency_level", ConsistencyLevelEnum.class, SearchParam.Builder::withConsistencyLevel);
@@ -59,8 +58,11 @@ public class MilvusSearchOpDispenser extends MilvusBaseOpDispenser<SearchParam>
ebF = op.enhanceFuncOptionally(ebF, "round_decimal", Integer.class, SearchParam.Builder::withRoundDecimal);
ebF = op.enhanceFuncOptionally(ebF, "ignore_growing", Boolean.class, SearchParam.Builder::withIgnoreGrowing);
ebF = op.enhanceFuncOptionally(ebF, "params", String.class, SearchParam.Builder::withParams);
ebF = op.enhanceFunc(ebF, "vector_field_name", String.class, SearchParam.Builder::withVectorFieldName);
ebF = op.enhanceFuncOptionally(ebF,"vectors",List.class,SearchParam.Builder::withVectors);
ebF = op.enhanceFunc(ebF, List.of("vector_field_name", "vector_field"), String.class,
SearchParam.Builder::withVectorFieldName);
// TODO: sanity check List of Floats vs List of List of Floats at func construction time.
ebF = op.enhanceFuncOptionally(ebF, "vectors", List.class, SearchParam.Builder::withVectors);
ebF = op.enhanceFuncOptionally(ebF, "vector", List.class, (b, l) -> b.withVectors(List.of(l)));
LongFunction<SearchParam.Builder> finalEbF = ebF;
return l -> finalEbF.apply(l).build();
}
@@ -72,7 +74,7 @@ public class MilvusSearchOpDispenser extends MilvusBaseOpDispenser<SearchParam>
ParsedOp op,
LongFunction<String> targetF
) {
return l -> new MilvusSearchOp(clientF.apply(l),paramF.apply(l));
return l -> new MilvusSearchOp(clientF.apply(l), paramF.apply(l));
}
}

4
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowCollectionsOpDispenser.java
View File

@@ -20,7 +20,7 @@ import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.ShowType;
import io.milvus.param.collection.ShowCollectionsParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.MilvusAdapterUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusShowCollectionsOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
@@ -44,7 +44,7 @@ public class MilvusShowCollectionsOpDispenser extends MilvusBaseOpDispenser<Show
) {
LongFunction<ShowCollectionsParam.Builder> ebF =
l -> ShowCollectionsParam.newBuilder();
LongFunction<List<String>> collectionsF = l -> MilvusUtils.splitNames(targetF.apply(l));
LongFunction<List<String>> collectionsF = l -> MilvusAdapterUtils.splitNames(targetF.apply(l));
LongFunction<ShowCollectionsParam.Builder> finalEbF = ebF;
ebF = l -> finalEbF.apply(l).withCollectionNames(collectionsF.apply(l));
ebF = op.enhanceFuncOptionally(ebF,List.of("database_name","database"),String.class,

5
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusShowPartitionsOpDispenser.java
View File

@@ -17,10 +17,9 @@
package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.collection.ShowCollectionsParam;
import io.milvus.param.partition.ShowPartitionsParam;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
import io.nosqlbench.adapter.milvus.MilvusUtils;
import io.nosqlbench.adapter.milvus.MilvusAdapterUtils;
import io.nosqlbench.adapter.milvus.ops.MilvusBaseOp;
import io.nosqlbench.adapter.milvus.ops.MilvusShowPartitionsOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
@@ -45,7 +44,7 @@ public class MilvusShowPartitionsOpDispenser extends MilvusBaseOpDispenser<ShowP
LongFunction<ShowPartitionsParam.Builder> ebF =
l -> ShowPartitionsParam.newBuilder();
LongFunction<List<String>> partitionsF = l -> MilvusUtils.splitNames(targetF.apply(l));
LongFunction<List<String>> partitionsF = l -> MilvusAdapterUtils.splitNames(targetF.apply(l));
LongFunction<ShowPartitionsParam.Builder> finalEbF = ebF;
ebF = l -> finalEbF.apply(l).withPartitionNames(partitionsF.apply(l));
ebF = op.enhanceFuncOptionally(ebF,List.of("collection_name","collection"),String.class,

70
adapter-milvus/src/main/resources/activities/milvus.yaml
View File

@@ -5,8 +5,11 @@ description: |
TEMPLATE(milvushost,localhost)
TEMPLATE(datafile)
TEMPLATE(database,baselines)
TEMPLATE(trainsize)
TEMPLATE(testsize)
TEMPLATE(collection,vector)
TEMPLATE(trainsize,1183514)
TEMPLATE(testsize,10000)
TEMPLATE(top_k,100)
TEMPLATE(filetype,hdf5)
schema: Install the schema required to run the test
rampup: Measure how long it takes to load a set of embeddings
@@ -26,11 +29,11 @@ scenarios:
uri=http://TEMPLATE(milvushost):19530 database=TEMPLATE(database) token=root:Milvus
rampup: >-
run tags==block:rampup errors=counter,warn
cycles===TEMPLATE(rampup_cycles,TEMPLATE(trainsize,100)) threads===TEMPLATE(rampup_threads,10)
cycles===TEMPLATE(rampup_cycles,TEMPLATE(trainsize)) threads===TEMPLATE(rampup_threads,auto)
uri=http://TEMPLATE(milvushost):19530 database=TEMPLATE(database) token=root:Milvus
search: >-
run tags==block:search errors=counter,warn
cycles===TEMPLATE(search_cycles,TEMPLATE(testsize,100)) threads===TEMPLATE(search_threads,10)
cycles===TEMPLATE(search_cycles,TEMPLATE(testsize)) threads===TEMPLATE(search_threads,auto)
uri=http://TEMPLATE(milvushost):19530 database=TEMPLATE(database) token=root:Milvus
params:
@@ -57,14 +60,14 @@ blocks:
drop:
ops:
# https://milvus.io/api-reference/java/v2.3.x/Collection/dropCollection().md
drop_col_op:
drop_collection: "TEMPLATE(collection,vector)"
database_name: "TEMPLATE(database)"
# https://milvus.io/api-reference/java/v2.3.x/Index/dropIndex().md
drop_index_op:
drop_index: "TEMPLATE(collection,vector)_TEMPLATE(vector_field,value)_idx"
collection_name: "TEMPLATE(collection,vector)"
database_name: "TEMPLATE(database)"
drop_index: "TEMPLATE(collection)_TEMPLATE(vector_field,value)_idx"
collection_name: "TEMPLATE(collection)"
# database_name: "TEMPLATE(database)"
drop_col_op:
drop_collection: "TEMPLATE(collection)"
# database_name: "TEMPLATE(database)"
# https://milvus.io/api-reference/java/v2.3.x/Index/dropIndex().md
drop_db_op:
drop_database: "TEMPLATE(database)"
schema:
@@ -73,7 +76,7 @@ blocks:
create_database: "TEMPLATE(database)"
# https://milvus.io/api-reference/java/v2.3.x/Collection/createCollection().md
create_col_op:
create_collection: "TEMPLATE(collection,vector)"
create_collection: "TEMPLATE(collection)"
description: "TEMPLATE(desc,a simple milvus/zilliz vector collection)"
consistency_level: "BOUNDED"
field_types:
@@ -92,8 +95,8 @@ blocks:
# https://milvus.io/api-reference/java/v2.3.x/Index/createIndex().md
create_index_op:
create_index: "TEMPLATE(collection,vector)_TEMPLATE(vector_field,value)_idx"
collection_name: "TEMPLATE(collection,vector)"
create_index: "TEMPLATE(collection)_TEMPLATE(vector_field,value)_idx"
collection_name: "TEMPLATE(collection)"
field_name: "TEMPLATE(vector_field,value)"
index_type: "TEMPLATE(idx_type,DISKANN)"
metric_type: "TEMPLATE(similarity_function,COSINE)"
@@ -103,24 +106,49 @@ blocks:
ops:
# https://milvus.io/api-reference/java/v2.3.x/Collection/insert().md
insert_op:
insert: "TEMPLATE(collection,vector)"
insert: "TEMPLATE(collection)"
rows:
key: "{row_key}"
value: "{train_floatlist_TEMPLATE(filetype,hdf5)}"
value: "{train_floatlist_TEMPLATE(filetype)}"
# fields:
# key: (List this) "{row_key}"
# value: (List this) "{train_floatlist_TEMPLATE(filetype,hdf5)}"
# value: (List this) "{train_floatlist_TEMPLATE(filetype)}"
search:
ops:
# https://milvus.io/api-reference/java/v2.3.x/High-level%20API/search().md
# https://milvus.io/api-reference/java/v2.3.x/Query%20and%20Search/search().md
search_op:
search: "TEMPLATE(collection,vector)"
vectors: "{test_floatlist_TEMPLATE(filetype,hdf5)}"
output_fields:
search: "TEMPLATE(collection)"
vector: "{test_floatlist_TEMPLATE(filetype)}"
metric_type: COSINE
out_fields:
- key
- value
vector_field_name: "value"
top_k: TEMPLATE(top_k,100)
top_k: TEMPLATE(top_k)
consistency_level: "TEMPLATE(read_cl,EVENTUALLY)"
verifier-init: |
relevancy= new io.nosqlbench.nb.api.engine.metrics.wrappers.RelevancyMeasures(_parsed_op);
for (int k in List.of(100)) {
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("recall",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("precision",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("F1",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("RR",k));
relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("AP",k));
}
# for (int k in List.of(1,2,3,5,10,25,50,75)) {
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("s_recall",k));
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("s_precision",k));
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("s_F1",k));
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("s_RR",k));
# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("s_AP",k));
# }
verifier: |
// driver-specific function
actual_indices=io.nosqlbench.adapter.milvus.MilvusUtils.intArrayFromMilvusSearchResults("key",result)
// driver-agnostic function
relevancy.accept({relevant_indices_TEMPLATE(filetype)},actual_indices);
// because we are "verifying" although this needs to be reorganized
return true;

57
adapter-milvus/src/test/java/io/nosqlbench/adapter/milvus/MilvusAdapterUtilsTest.java Normal file
View File

@@ -0,0 +1,57 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.milvus;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class MilvusAdapterUtilsTest {
@Test
void testSplitNames() {
assertThrows(AssertionError.class, () -> MilvusAdapterUtils.splitNames(null));
assertThrows(AssertionError.class, () -> MilvusAdapterUtils.splitNames(""));
assertEquals(List.of("abc"), MilvusAdapterUtils.splitNames("abc"));
assertEquals(List.of("abc", "def"), MilvusAdapterUtils.splitNames("abc def"));
assertEquals(List.of("abc", "def"), MilvusAdapterUtils.splitNames("abc,def"));
assertEquals(List.of("abc", "def"), MilvusAdapterUtils.splitNames("abc , def"));
}
@Test
void testSplitLongs() {
assertThrows(AssertionError.class, () -> MilvusAdapterUtils.splitLongs(null));
assertThrows(AssertionError.class, () -> MilvusAdapterUtils.splitLongs(""));
assertEquals(List.of(123L), MilvusAdapterUtils.splitLongs("123"));
assertEquals(List.of(123L, 456L), MilvusAdapterUtils.splitLongs("123 456"));
assertEquals(List.of(123L, 456L), MilvusAdapterUtils.splitLongs("123,456"));
assertEquals(List.of(123L, 456L), MilvusAdapterUtils.splitLongs("123 , 456"));
assertThrows(NumberFormatException.class, () -> MilvusAdapterUtils.splitLongs("abc"));
}
@Test
void testMaskDigits() {
assertThrows(AssertionError.class, () -> MilvusAdapterUtils.maskDigits(""));
assertThrows(AssertionError.class, () -> MilvusAdapterUtils.maskDigits(null));
assertEquals("abc", MilvusAdapterUtils.maskDigits("abc"));
assertEquals("***", MilvusAdapterUtils.maskDigits("123"));
assertEquals("abc***def", MilvusAdapterUtils.maskDigits("abc123def"));
}
}

57
adapter-milvus/src/test/java/io/nosqlbench/adapter/milvus/MilvusUtilsTest.java
View File

@@ -1,57 +0,0 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.milvus;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class MilvusUtilsTest {
@Test
void testSplitNames() {
assertThrows(AssertionError.class, () -> MilvusUtils.splitNames(null));
assertThrows(AssertionError.class, () -> MilvusUtils.splitNames(""));
assertEquals(List.of("abc"), MilvusUtils.splitNames("abc"));
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc def"));
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc,def"));
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc , def"));
}
@Test
void testSplitLongs() {
assertThrows(AssertionError.class, () -> MilvusUtils.splitLongs(null));
assertThrows(AssertionError.class, () -> MilvusUtils.splitLongs(""));
assertEquals(List.of(123L), MilvusUtils.splitLongs("123"));
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123 456"));
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123,456"));
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123 , 456"));
assertThrows(NumberFormatException.class, () -> MilvusUtils.splitLongs("abc"));
}
@Test
void testMaskDigits() {
assertThrows(AssertionError.class, () -> MilvusUtils.maskDigits(""));
assertThrows(AssertionError.class, () -> MilvusUtils.maskDigits(null));
assertEquals("abc", MilvusUtils.maskDigits("abc"));
assertEquals("***", MilvusUtils.maskDigits("123"));
assertEquals("abc***def", MilvusUtils.maskDigits("abc123def"));
}
}

2
adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
View File

@@ -926,7 +926,7 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
BiFunction<FA, FE, FA> combiner
) {
for (String field : fields) {
if (isDynamic(field)) {
if (isDefined(field)) {
LongFunction<FE> fieldEnhancerFunc = getAsRequiredFunction(field, type);
LongFunction<FA> lfa = l -> combiner.apply(func.apply(l), fieldEnhancerFunc.apply(l));
return lfa;