IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-12-23 15:40:44 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1926 from nosqlbench/milvus_updates

Browse Source 
Unit test failure fixes
This commit is contained in:
Madhavan 2024-04-15 09:07:35 -04:00 committed by GitHub
commit 4c30fd7cf6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 69 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
nb-adapters/adapter-milvus/pom.xml
View File

@ -50,12 +50,12 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>3.24.0</version>
<version>3.25.3</version>
</dependency>
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.3.4</version>
<version>2.3.5</version>
</dependency>
</dependencies>

19
nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java
View File

@ -51,10 +51,8 @@ public class MilvusSpace implements AutoCloseable {
* Create a new MilvusSpace Object which stores all stateful contextual information needed to interact
* with the Milvus/Zilliz database instance.
*
* @param name
* The name of this space
* @param cfg
* The configuration ({@link NBConfiguration}) for this nb run
* @param name The name of this space
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
*/
public MilvusSpace(String name, NBConfiguration cfg) {
this.name = name;
@ -89,16 +87,13 @@ public class MilvusSpace implements AutoCloseable {
).orElseGet(
() -> cfg.getOptional("token")
.orElseThrow(() -> new RuntimeException("You must provide either a token_file or a token to " +
"configure a Milvus client"))
"configure a Milvus/Zilliz client"))
);
builder = builder.withToken(requiredToken);
ConnectParam connectParams = builder.build();
logger.info(this.name + ": Creating new Milvus/Zilliz Client with (masked) " +
"token [" + MilvusAdapterUtils.maskDigits(builder.getToken()) + "], uri/endpoint [" + builder.getUri() + "]"
);
logger.info("{}: Creating new Milvus/Zilliz Client with (masked) token [{}], uri/endpoint [{}]",
this.name, MilvusAdapterUtils.maskDigits(builder.getToken()), builder.getUri());
return new MilvusServiceClient(connectParams);
}
@ -117,8 +112,8 @@ public class MilvusSpace implements AutoCloseable {
.setDescription("the URI endpoint in which the database is running.")
)
.add(
Param.optional(List.of("database_name","database"))
.setDescription("the name of the database to use. Defaults to 'baselines'")
Param.optional(List.of("database_name", "database"))
.setDescription("the name of the database to use.")
)
.asReadOnly();
}

9
nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java
View File

@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.grpc.DataType;
import io.milvus.param.collection.CollectionSchemaParam;
import io.milvus.param.collection.CreateCollectionParam;
import io.milvus.param.collection.FieldType;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
@ -70,11 +71,10 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser<Cre
CreateCollectionParam.Builder::withDatabaseName);
List<FieldType> fieldTypes = buildFieldTypesStruct(
op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey),
ebF
op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey)
);
final LongFunction<CreateCollectionParam.Builder> f = ebF;
ebF = l -> f.apply(l).withFieldTypes(fieldTypes);
ebF = l -> f.apply(l).withSchema(CollectionSchemaParam.newBuilder().withFieldTypes(fieldTypes).build());
final LongFunction<CreateCollectionParam.Builder> lastF = ebF;
return l -> lastF.apply(l).build();
@ -95,10 +95,9 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser<Cre
* Function to build the {@link FieldType}s for the {@link CreateCollectionParam}.
*
* @param fieldTypesData The static map of config data from the create collection request
* @param ebF
* @return a list of static field types
*/
private List<FieldType> buildFieldTypesStruct(Map<String, ParsedOp> fieldTypesData, LongFunction<CreateCollectionParam.Builder> ebF) {
private List<FieldType> buildFieldTypesStruct(Map<String, ParsedOp> fieldTypesData) {
List<FieldType> fieldTypes = new ArrayList<>();
fieldTypesData.forEach((name, fieldspec) -> {
FieldType.Builder builder = FieldType.newBuilder()

32
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/scheduling/TimeoutPredicateTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2024 nosqlbench
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,11 +26,11 @@ public class TimeoutPredicateTest {
@Test
public void testNeverCompletablePreciate() {
int interval=10;
int timeout=500;
int interval = 10;
int timeout = 500;
TimeoutPredicate<Boolean> wontMakeIt = TimeoutPredicate.of(
()->false,
() -> false,
l -> l,
Duration.ofMillis(timeout),
Duration.ofMillis(interval),
@ -43,12 +43,12 @@ public class TimeoutPredicateTest {
assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.pending);
resultNow = wontMakeIt.test();
assertThat(resultNow.duration_ns()).isBetween(10*1_000_000L,50*1_000_000L);
assertThat(resultNow.duration_ns()).isBetween(10 * 1_000_000L, 50 * 1_000_000_000L);
assertThat(resultNow.value()).isFalse();
assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.pending);
while (resultNow.status()== TimeoutPredicate.Status.pending) {
resultNow=wontMakeIt.test();
while (resultNow.status() == TimeoutPredicate.Status.pending) {
resultNow = wontMakeIt.test();
}
assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.incomplete);
@ -57,10 +57,10 @@ public class TimeoutPredicateTest {
@Test
public void testImmediatelyCompletablePreciate() {
int interval=10;
int timeout=5000;
int interval = 10;
int timeout = 5000;
TimeoutPredicate<Boolean> canMakeIt = TimeoutPredicate.of(
()->true,
() -> true,
l -> l,
Duration.ofMillis(timeout),
Duration.ofMillis(interval),
@ -77,13 +77,13 @@ public class TimeoutPredicateTest {
@Test
public void testEventuallyCompletePredicate() {
int interval=250;
int timeout=5000;
int interval = 250;
int timeout = 5000;
long now = System.currentTimeMillis();
long inASec = now+1000;
long inASec = now + 1000;
TimeoutPredicate<Long> canMakeIt = TimeoutPredicate.of(
System::currentTimeMillis,
l -> l>inASec,
l -> l > inASec,
Duration.ofMillis(timeout),
Duration.ofMillis(interval),
true
@ -92,9 +92,9 @@ public class TimeoutPredicateTest {
TimeoutPredicate.Result<Long> result = canMakeIt.test();
System.out.println(result);
while (result.status()== TimeoutPredicate.Status.pending) {
while (result.status() == TimeoutPredicate.Status.pending) {
// canMakeIt.blockUntilNextInterval();
result=canMakeIt.test();
result = canMakeIt.test();
System.out.println(canMakeIt);
System.out.println(result);
}

20
nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTemplateVarTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022-2023 nosqlbench
* Copyright (c) 2022-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,36 +30,36 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
@Test
public void testMultipleOccurencesOfSameTemplateVar() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars" }, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"),cmds.get(0).getArgMap());
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"), cmds.get(0).getArgMap());
OpTemplate optpl1 = workload1.getOps(true).get(0);
System.out.println("op from cmd1:"+optpl1);
System.out.println("op from cmd1:" + optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArgValue("workload"),cmds.get(1).getArgMap());
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArgValue("workload"), cmds.get(1).getArgMap());
OpTemplate optpl2 = workload2.getOps(true).get(0);
System.out.println("op from cmd2:"+optpl2);
System.out.println("op from cmd2:" + optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
}
@Test
public void testThatCLIOverridesWorkForTemplateVars() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars", "tvar1=overridden" }, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars", "tvar1=overridden"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"),cmds.get(0).getArgMap());
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"), cmds.get(0).getArgMap());
OpTemplate optpl1 = workload1.getOps(true).get(0);
System.out.println("op from cmd1:"+optpl1);
System.out.println("op from cmd1:" + optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
}
@Test
public void testThatAdditionalCLIParamIsAdded() {
NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
assertThat(cmds).hasSize(2);

10
nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022-2023 nosqlbench
* Copyright (c) 2022-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,7 +33,7 @@ public class NBCLIScenarioPreprocessorTest {
@Test
public void providePathForScenario() {
NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
}
@ -105,7 +105,7 @@ public class NBCLIScenarioPreprocessorTest {
List<Cmd> cmds = opts.getCommands();
assertThat(cmds.size()).isEqualTo(1);
assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
"_impl","run",
"_impl", "run",
"alias", "with_template",
"container", "template_test",
"cycles", "20",
@ -178,7 +178,7 @@ public class NBCLIScenarioPreprocessorTest {
List<Cmd> cmds = opts.getCommands();
assertThat(cmds.size()).isEqualTo(1);
assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
"_impl","run",
"_impl", "run",
"alias", "schema",
"container", "schema_only",
"cycles-test", "20",
@ -188,7 +188,7 @@ public class NBCLIScenarioPreprocessorTest {
"tags", "block:\"schema.*\"",
"workload", "scenario_test"
));
NBCLIOptions opts1 = new NBCLIOptions(new String[]{"local/example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts1 = new NBCLIOptions(new String[]{"example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds1 = opts1.getCommands();
assertThat(cmds1.size()).isEqualTo(1);
assertThat(cmds1.get(0).getArgValueOrNull("cycles_test")).isNull();

15
nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios.yaml Normal file
View File

@ -0,0 +1,15 @@
# example_scenarios.yaml
scenarios:
default:
one: run cycles=3 alias=A driver=stdout
two: run cycles=5 alias=B driver=stdout
namedsteps:
one: run cycles=3 alias=A driver=stdout testparam1=testvalue1
two: run cycles=5 alias=B driver=stdout
bindings:
cycle: Identity()
name: NumberNameToCycle()
ops:
cycle: "cycle {cycle}\n"

10
nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml Normal file
View File

@ -0,0 +1,10 @@
# example_scenarios_templatevars.yaml
scenarios:
default:
first: run cycles=3 alias=A driver=stdout tvar1=replaced
second: run cycles=5 alias=B driver=stdout
bindings:
cycle: Identity()
name: NumberNameToCycle()
ops:
cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n"