mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
minor changes
This commit is contained in:
14
.run/milvus_vector2__drop__glove_25.run.xml
Normal file
14
.run/milvus_vector2__drop__glove_25.run.xml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
<component name="ProjectRunConfigurationManager">
|
||||||
|
<configuration default="false" name="milvus_vector2__drop__glove_25" type="JarApplication" folderName="Milvus">
|
||||||
|
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
|
||||||
|
<option name="credential" />
|
||||||
|
<option name="region" />
|
||||||
|
<option name="useCurrentConnection" value="false" />
|
||||||
|
</extension>
|
||||||
|
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
|
||||||
|
<option name="PROGRAM_PARAMETERS" value="milvus default.drop --show-stacktraces dimensions=25 testsize=10000 trainsize=100000 datafile=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=cosine --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25"" />
|
||||||
|
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/milvus" />
|
||||||
|
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
|
||||||
|
<method v="2" />
|
||||||
|
</configuration>
|
||||||
|
</component>
|
||||||
@@ -53,7 +53,7 @@ public class MilvusOpMapper implements OpMapper<MilvusBaseOp<?>> {
|
|||||||
"type",
|
"type",
|
||||||
"target"
|
"target"
|
||||||
);
|
);
|
||||||
logger.info(() -> "Using " + typeAndTarget.enumId + " statement form for '" + op.getName());
|
logger.info(() -> "Using '" + typeAndTarget.enumId + "' statement form for '" + op.getName() + "'");
|
||||||
|
|
||||||
return switch (typeAndTarget.enumId) {
|
return switch (typeAndTarget.enumId) {
|
||||||
case drop_collection -> new MilvusDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case drop_collection -> new MilvusDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
@@ -61,7 +61,7 @@ public class MilvusOpMapper implements OpMapper<MilvusBaseOp<?>> {
|
|||||||
case create_index -> new MilvusCreateIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case create_index -> new MilvusCreateIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case drop_index -> new MilvusDropIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case drop_index -> new MilvusDropIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case insert -> new MilvusInsertOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case insert -> new MilvusInsertOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case delete -> new MilvusDeleteOpDispenser(adapter,op,typeAndTarget.targetFunction);
|
case delete -> new MilvusDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case search -> new MilvusSearchOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case search -> new MilvusSearchOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case alter_alias -> new MilvusAlterAliasOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case alter_alias -> new MilvusAlterAliasOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case alter_collection -> new MilvusAlterCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case alter_collection -> new MilvusAlterCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
@@ -74,7 +74,8 @@ public class MilvusOpMapper implements OpMapper<MilvusBaseOp<?>> {
|
|||||||
case create_database -> new MilvusCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case create_database -> new MilvusCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case query -> new MilvusQueryOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case query -> new MilvusQueryOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case delete_credential -> new MilvusDeleteCredentialOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case delete_credential -> new MilvusDeleteCredentialOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case describe_collection -> new MilvusDescribeCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case describe_collection ->
|
||||||
|
new MilvusDescribeCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case describe_index -> new MilvusDescribeIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case describe_index -> new MilvusDescribeIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case drop_alias -> new MilvusDropAliasOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case drop_alias -> new MilvusDropAliasOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
case get_metrics -> new MilvusGetMetricsOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
case get_metrics -> new MilvusGetMetricsOpDispenser(adapter, op, typeAndTarget.targetFunction);
|
||||||
|
|||||||
@@ -16,32 +16,40 @@
|
|||||||
|
|
||||||
package io.nosqlbench.adapter.milvus;
|
package io.nosqlbench.adapter.milvus;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
public class MilvusUtils {
|
public class MilvusUtils {
|
||||||
|
|
||||||
public static List<String> splitNames(String input) {
|
public static List<String> splitNames(String input) {
|
||||||
Objects.requireNonNull(input);
|
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
|
||||||
return Arrays.asList(input.split("( +| *, *)"));
|
return Arrays.stream(input.split("( +| *, *)"))
|
||||||
|
.filter(StringUtils::isNotBlank)
|
||||||
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<Long> splitLongs(String input) {
|
public static List<Long> splitLongs(String input) {
|
||||||
Objects.requireNonNull(input);
|
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
|
||||||
return Arrays.stream(input.split("( +| *, *)")).map(Long::parseLong).toList();
|
return Arrays.stream(input.split("( +| *, *)"))
|
||||||
|
.filter(StringUtils::isNotBlank)
|
||||||
|
.map(Long::parseLong)
|
||||||
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mask the digits in the given string with '*'
|
* Mask the digits in the given string with '*'
|
||||||
|
*
|
||||||
* @param unmasked The string to mask
|
* @param unmasked The string to mask
|
||||||
* @return The masked string
|
* @return The masked string
|
||||||
*/
|
*/
|
||||||
protected static String maskDigits(String unmasked) {
|
protected static String maskDigits(String unmasked) {
|
||||||
int inputLength = (null == unmasked) ? 0 : unmasked.length();
|
assert StringUtils.isNotBlank(unmasked) && StringUtils.isNotEmpty(unmasked);
|
||||||
|
int inputLength = unmasked.length();
|
||||||
StringBuilder masked = new StringBuilder(inputLength);
|
StringBuilder masked = new StringBuilder(inputLength);
|
||||||
for(char ch : unmasked.toCharArray()) {
|
for (char ch : unmasked.toCharArray()) {
|
||||||
if (Character.isDigit(ch)) {
|
if (Character.isDigit(ch)) {
|
||||||
masked.append("*");
|
masked.append("*");
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -29,23 +29,17 @@ import org.apache.logging.log4j.Logger;
|
|||||||
|
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
import static org.matheclipse.core.expression.S.l;
|
|
||||||
|
|
||||||
public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenser<DropCollectionParam> {
|
public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenser<DropCollectionParam> {
|
||||||
private static final Logger logger = LogManager.getLogger(MilvusDropCollectionOpDispenser.class);
|
private static final Logger logger = LogManager.getLogger(MilvusDropCollectionOpDispenser.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <P>Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}.</P>
|
* <P>Create a new {@link MilvusDropCollectionOpDispenser} subclassed from {@link MilvusBaseOpDispenser}.</P>
|
||||||
*
|
*
|
||||||
* <P>{@see <A HREF="https://milvus.io/docs/drop_collection.md">Drop Collection</A>}</P>
|
* <P>{@see <A HREF="https://milvus.io/docs/drop_collection.md">Drop Collection</A>}</P>
|
||||||
*
|
*
|
||||||
* @param adapter
|
* @param adapter The associated {@link MilvusDriverAdapter}
|
||||||
* The associated {@link MilvusDriverAdapter}
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
* @param op
|
* @param targetFunction A LongFunction that returns the specified Milvus Index for this Op
|
||||||
* The {@link ParsedOp} encapsulating the activity for this cycle
|
|
||||||
* @param targetFunction
|
|
||||||
* A LongFunction that returns the specified Milvus Index for this Op
|
|
||||||
*
|
|
||||||
*/
|
*/
|
||||||
public MilvusDropCollectionOpDispenser(MilvusDriverAdapter adapter,
|
public MilvusDropCollectionOpDispenser(MilvusDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
@@ -54,18 +48,23 @@ public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenser<DropC
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public LongFunction<DropCollectionParam> getParamFunc(LongFunction<MilvusServiceClient> clientF, ParsedOp op, LongFunction<String> targetF) {
|
public LongFunction<DropCollectionParam> getParamFunc(
|
||||||
|
LongFunction<MilvusServiceClient> clientF,
|
||||||
|
ParsedOp op,
|
||||||
|
LongFunction<String> targetF) {
|
||||||
LongFunction<DropCollectionParam.Builder> f =
|
LongFunction<DropCollectionParam.Builder> f =
|
||||||
l -> DropCollectionParam.newBuilder().withCollectionName(targetF.apply(l));
|
l -> DropCollectionParam.newBuilder().withCollectionName(targetF.apply(l));
|
||||||
f = op.enhanceFuncOptionally(f,"database_name",String.class,DropCollectionParam.Builder::withDatabaseName);
|
f = op.enhanceFuncOptionally(f, "database_name", String.class, DropCollectionParam.Builder::withDatabaseName);
|
||||||
LongFunction<DropCollectionParam.Builder> finalF = f;
|
LongFunction<DropCollectionParam.Builder> finalF = f;
|
||||||
return l -> finalF.apply(l).build();
|
return l -> finalF.apply(l).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public LongFunction<MilvusBaseOp<DropCollectionParam>> createOpFunc(LongFunction<DropCollectionParam> paramF, LongFunction<MilvusServiceClient> clientF, ParsedOp op, LongFunction<String> targetF) {
|
public LongFunction<MilvusBaseOp<DropCollectionParam>> createOpFunc(LongFunction<DropCollectionParam> paramF,
|
||||||
|
LongFunction<MilvusServiceClient> clientF,
|
||||||
|
ParsedOp op, LongFunction<String> targetF) {
|
||||||
|
|
||||||
return l -> new MilvusDropCollectionOp(clientF.apply(l),paramF.apply(l));
|
return l -> new MilvusDropCollectionOp(clientF.apply(l), paramF.apply(l));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,18 +16,18 @@ scenarios:
|
|||||||
default:
|
default:
|
||||||
drop: >-
|
drop: >-
|
||||||
run tags==block:drop errors===stop cycles===UNDEF threads===UNDEF
|
run tags==block:drop errors===stop cycles===UNDEF threads===UNDEF
|
||||||
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
|
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
|
||||||
schema: >-
|
schema: >-
|
||||||
run tags==block:schema errors===stop cycles===2 threads===1
|
run tags==block:schema errors===stop cycles===2 threads===1
|
||||||
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
|
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
|
||||||
rampup: >-
|
rampup: >-
|
||||||
run tags==block:rampup errors=counter,warn
|
run tags==block:rampup errors=counter,warn
|
||||||
cycles===TEMPLATE(rampup_cycles,100) threads===TEMPLATE(rampup_threads,10)
|
cycles===TEMPLATE(trainsize,100) threads===TEMPLATE(rampup_threads,10)
|
||||||
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
|
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
|
||||||
search: >-
|
search: >-
|
||||||
run tags==block:search errors=counter,warn
|
run tags==block:search errors=counter,warn
|
||||||
cycles===TEMPLATE(search_cycles,100) threads===TEMPLATE(search_threads,10)
|
cycles===TEMPLATE(testsize,100) threads===TEMPLATE(search_threads,10)
|
||||||
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
|
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
|
||||||
|
|
||||||
params:
|
params:
|
||||||
driver: milvus
|
driver: milvus
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2024 nosqlbench
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.nosqlbench.adapter.milvus;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
|
class MilvusUtilsTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSplitNames() {
|
||||||
|
assertThrows(AssertionError.class, () -> MilvusUtils.splitNames(null));
|
||||||
|
assertThrows(AssertionError.class, () -> MilvusUtils.splitNames(""));
|
||||||
|
assertEquals(List.of("abc"), MilvusUtils.splitNames("abc"));
|
||||||
|
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc def"));
|
||||||
|
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc,def"));
|
||||||
|
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc , def"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSplitLongs() {
|
||||||
|
assertThrows(AssertionError.class, () -> MilvusUtils.splitLongs(null));
|
||||||
|
assertThrows(AssertionError.class, () -> MilvusUtils.splitLongs(""));
|
||||||
|
assertEquals(List.of(123L), MilvusUtils.splitLongs("123"));
|
||||||
|
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123 456"));
|
||||||
|
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123,456"));
|
||||||
|
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123 , 456"));
|
||||||
|
assertThrows(NumberFormatException.class, () -> MilvusUtils.splitLongs("abc"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMaskDigits() {
|
||||||
|
assertThrows(AssertionError.class, () -> MilvusUtils.maskDigits(""));
|
||||||
|
assertThrows(AssertionError.class, () -> MilvusUtils.maskDigits(null));
|
||||||
|
assertEquals("abc", MilvusUtils.maskDigits("abc"));
|
||||||
|
assertEquals("***", MilvusUtils.maskDigits("123"));
|
||||||
|
assertEquals("abc***def", MilvusUtils.maskDigits("abc123def"));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user