IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-01-26 15:36:33 -06:00
Code Issues Packages Projects Releases Wiki Activity

minor changes

Browse Source
This commit is contained in:
Madhavan Sridharan 2024-03-20 17:57:25 -04:00
parent 2b87e92e81
commit f6e92f77aa
6 changed files with 109 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.run/milvus_vector2__drop__glove_25.run.xml Normal file
View File

@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="milvus_vector2__drop__glove_25" type="JarApplication" folderName="Milvus">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="milvus default.drop --show-stacktraces dimensions=25 testsize=10000 trainsize=100000 datafile=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=cosine --progress console:1s -v --add-labels &quot;dimensions:25,dataset=glove-25&quot;" />
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/milvus" />
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
<method v="2" />
</configuration>
</component>

7
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusOpMapper.java
View File

@ -53,7 +53,7 @@ public class MilvusOpMapper implements OpMapper<MilvusBaseOp<?>> {
"type",
"target"
);
logger.info(() -> "Using " + typeAndTarget.enumId + " statement form for '" + op.getName());
logger.info(() -> "Using '" + typeAndTarget.enumId + "' statement form for '" + op.getName() + "'");
return switch (typeAndTarget.enumId) {
case drop_collection -> new MilvusDropCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
@ -61,7 +61,7 @@ public class MilvusOpMapper implements OpMapper<MilvusBaseOp<?>> {
case create_index -> new MilvusCreateIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
case drop_index -> new MilvusDropIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
case insert -> new MilvusInsertOpDispenser(adapter, op, typeAndTarget.targetFunction);
case delete -> new MilvusDeleteOpDispenser(adapter,op,typeAndTarget.targetFunction);
case delete -> new MilvusDeleteOpDispenser(adapter, op, typeAndTarget.targetFunction);
case search -> new MilvusSearchOpDispenser(adapter, op, typeAndTarget.targetFunction);
case alter_alias -> new MilvusAlterAliasOpDispenser(adapter, op, typeAndTarget.targetFunction);
case alter_collection -> new MilvusAlterCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
@ -74,7 +74,8 @@ public class MilvusOpMapper implements OpMapper<MilvusBaseOp<?>> {
case create_database -> new MilvusCreateDatabaseOpDispenser(adapter, op, typeAndTarget.targetFunction);
case query -> new MilvusQueryOpDispenser(adapter, op, typeAndTarget.targetFunction);
case delete_credential -> new MilvusDeleteCredentialOpDispenser(adapter, op, typeAndTarget.targetFunction);
case describe_collection -> new MilvusDescribeCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
case describe_collection ->
new MilvusDescribeCollectionOpDispenser(adapter, op, typeAndTarget.targetFunction);
case describe_index -> new MilvusDescribeIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
case drop_alias -> new MilvusDropAliasOpDispenser(adapter, op, typeAndTarget.targetFunction);
case get_metrics -> new MilvusGetMetricsOpDispenser(adapter, op, typeAndTarget.targetFunction);

22
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusUtils.java
View File

@ -16,32 +16,40 @@
package io.nosqlbench.adapter.milvus;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
public class MilvusUtils {
public static List<String> splitNames(String input) {
Objects.requireNonNull(input);
return Arrays.asList(input.split("( +| *, *)"));
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
return Arrays.stream(input.split("( +| *, *)"))
.filter(StringUtils::isNotBlank)
.toList();
}
public static List<Long> splitLongs(String input) {
Objects.requireNonNull(input);
return Arrays.stream(input.split("( +| *, *)")).map(Long::parseLong).toList();
assert StringUtils.isNotBlank(input) && StringUtils.isNotEmpty(input);
return Arrays.stream(input.split("( +| *, *)"))
.filter(StringUtils::isNotBlank)
.map(Long::parseLong)
.toList();
}
/**
* Mask the digits in the given string with '*'
*
* @param unmasked The string to mask
* @return The masked string
*/
protected static String maskDigits(String unmasked) {
int inputLength = (null == unmasked) ? 0 : unmasked.length();
assert StringUtils.isNotBlank(unmasked) && StringUtils.isNotEmpty(unmasked);
int inputLength = unmasked.length();
StringBuilder masked = new StringBuilder(inputLength);
for(char ch : unmasked.toCharArray()) {
for (char ch : unmasked.toCharArray()) {
if (Character.isDigit(ch)) {
masked.append("*");
} else {

27
adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusDropCollectionOpDispenser.java
View File

@ -29,23 +29,17 @@ import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
import static org.matheclipse.core.expression.S.l;
public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenser<DropCollectionParam> {
private static final Logger logger = LogManager.getLogger(MilvusDropCollectionOpDispenser.class);
/**
* <P>Create a new MilvusDeleteOpDispenser subclassed from {@link MilvusBaseOpDispenser}.</P>
* <P>Create a new {@link MilvusDropCollectionOpDispenser} subclassed from {@link MilvusBaseOpDispenser}.</P>
*
* <P>{@see <A HREF="https://milvus.io/docs/drop_collection.md">Drop Collection</A>}</P>
*
* @param adapter
* The associated {@link MilvusDriverAdapter}
* @param op
* The {@link ParsedOp} encapsulating the activity for this cycle
* @param targetFunction
* A LongFunction that returns the specified Milvus Index for this Op
*
* @param adapter The associated {@link MilvusDriverAdapter}
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
* @param targetFunction A LongFunction that returns the specified Milvus Index for this Op
*/
public MilvusDropCollectionOpDispenser(MilvusDriverAdapter adapter,
ParsedOp op,
@ -54,18 +48,23 @@ public class MilvusDropCollectionOpDispenser extends MilvusBaseOpDispenser<DropC
}
@Override
public LongFunction<DropCollectionParam> getParamFunc(LongFunction<MilvusServiceClient> clientF, ParsedOp op, LongFunction<String> targetF) {
public LongFunction<DropCollectionParam> getParamFunc(
LongFunction<MilvusServiceClient> clientF,
ParsedOp op,
LongFunction<String> targetF) {
LongFunction<DropCollectionParam.Builder> f =
l -> DropCollectionParam.newBuilder().withCollectionName(targetF.apply(l));
f = op.enhanceFuncOptionally(f,"database_name",String.class,DropCollectionParam.Builder::withDatabaseName);
f = op.enhanceFuncOptionally(f, "database_name", String.class, DropCollectionParam.Builder::withDatabaseName);
LongFunction<DropCollectionParam.Builder> finalF = f;
return l -> finalF.apply(l).build();
}
@Override
public LongFunction<MilvusBaseOp<DropCollectionParam>> createOpFunc(LongFunction<DropCollectionParam> paramF, LongFunction<MilvusServiceClient> clientF, ParsedOp op, LongFunction<String> targetF) {
public LongFunction<MilvusBaseOp<DropCollectionParam>> createOpFunc(LongFunction<DropCollectionParam> paramF,
LongFunction<MilvusServiceClient> clientF,
ParsedOp op, LongFunction<String> targetF) {
return l -> new MilvusDropCollectionOp(clientF.apply(l),paramF.apply(l));
return l -> new MilvusDropCollectionOp(clientF.apply(l), paramF.apply(l));
}
}

12
adapter-milvus/src/main/resources/activities/milvus.yaml
View File

@ -16,18 +16,18 @@ scenarios:
default:
drop: >-
run tags==block:drop errors===stop cycles===UNDEF threads===UNDEF
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
schema: >-
run tags==block:schema errors===stop cycles===2 threads===1
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
rampup: >-
run tags==block:rampup errors=counter,warn
cycles===TEMPLATE(rampup_cycles,100) threads===TEMPLATE(rampup_threads,10)
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
cycles===TEMPLATE(trainsize,100) threads===TEMPLATE(rampup_threads,10)
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
search: >-
run tags==block:search errors=counter,warn
cycles===TEMPLATE(search_cycles,100) threads===TEMPLATE(search_threads,10)
database_name=baselines uri=http://TEMPLATE(milvushost):19530 token=root:Milvus
cycles===TEMPLATE(testsize,100) threads===TEMPLATE(search_threads,10)
database_name=baselines uri="http://TEMPLATE(milvushost):19530" token=root:Milvus
params:
driver: milvus

57
adapter-milvus/src/test/java/io/nosqlbench/adapter/milvus/MilvusUtilsTest.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.milvus;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class MilvusUtilsTest {
@Test
void testSplitNames() {
assertThrows(AssertionError.class, () -> MilvusUtils.splitNames(null));
assertThrows(AssertionError.class, () -> MilvusUtils.splitNames(""));
assertEquals(List.of("abc"), MilvusUtils.splitNames("abc"));
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc def"));
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc,def"));
assertEquals(List.of("abc", "def"), MilvusUtils.splitNames("abc , def"));
}
@Test
void testSplitLongs() {
assertThrows(AssertionError.class, () -> MilvusUtils.splitLongs(null));
assertThrows(AssertionError.class, () -> MilvusUtils.splitLongs(""));
assertEquals(List.of(123L), MilvusUtils.splitLongs("123"));
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123 456"));
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123,456"));
assertEquals(List.of(123L, 456L), MilvusUtils.splitLongs("123 , 456"));
assertThrows(NumberFormatException.class, () -> MilvusUtils.splitLongs("abc"));
}
@Test
void testMaskDigits() {
assertThrows(AssertionError.class, () -> MilvusUtils.maskDigits(""));
assertThrows(AssertionError.class, () -> MilvusUtils.maskDigits(null));
assertEquals("abc", MilvusUtils.maskDigits("abc"));
assertEquals("***", MilvusUtils.maskDigits("123"));
assertEquals("abc***def", MilvusUtils.maskDigits("abc123def"));
}
}