IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1926 from nosqlbench/milvus_updates

Browse Source 
Unit test failure fixes
This commit is contained in:
Madhavan 2024-04-15 09:07:35 -04:00 committed by GitHub
commit 4c30fd7cf6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 69 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
nb-adapters/adapter-milvus/pom.xml
View File

@ -50,12 +50,12 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>3.24.0</version>
<version>3.25.3</version>
</dependency>
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.3.4</version>
<version>2.3.5</version>
</dependency>
</dependencies>

17
nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java
View File

@ -51,10 +51,8 @@ public class MilvusSpace implements AutoCloseable {
* Create a new MilvusSpace Object which stores all stateful contextual information needed to interact
* with the Milvus/Zilliz database instance.
*
* @param name
* The name of this space
* @param cfg
* The configuration ({@link NBConfiguration}) for this nb run
* @param name The name of this space
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
*/
public MilvusSpace(String name, NBConfiguration cfg) {
this.name = name;
@ -89,16 +87,13 @@ public class MilvusSpace implements AutoCloseable {
).orElseGet(
() -> cfg.getOptional("token")
.orElseThrow(() -> new RuntimeException("You must provide either a token_file or a token to " +
"configure a Milvus client"))
"configure a Milvus/Zilliz client"))
);
builder = builder.withToken(requiredToken);
ConnectParam connectParams = builder.build();
logger.info(this.name + ": Creating new Milvus/Zilliz Client with (masked) " +
"token [" + MilvusAdapterUtils.maskDigits(builder.getToken()) + "], uri/endpoint [" + builder.getUri() + "]"
);
logger.info("{}: Creating new Milvus/Zilliz Client with (masked) token [{}], uri/endpoint [{}]",
this.name, MilvusAdapterUtils.maskDigits(builder.getToken()), builder.getUri());
return new MilvusServiceClient(connectParams);
}
@ -118,7 +113,7 @@ public class MilvusSpace implements AutoCloseable {
)
.add(
Param.optional(List.of("database_name", "database"))
.setDescription("the name of the database to use. Defaults to 'baselines'")
.setDescription("the name of the database to use.")
)
.asReadOnly();
}

9
nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java
View File

@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers;
import io.milvus.client.MilvusServiceClient;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.grpc.DataType;
import io.milvus.param.collection.CollectionSchemaParam;
import io.milvus.param.collection.CreateCollectionParam;
import io.milvus.param.collection.FieldType;
import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
@ -70,11 +71,10 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser<Cre
CreateCollectionParam.Builder::withDatabaseName);
List<FieldType> fieldTypes = buildFieldTypesStruct(
op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey),
ebF
op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey)
);
final LongFunction<CreateCollectionParam.Builder> f = ebF;
ebF = l -> f.apply(l).withFieldTypes(fieldTypes);
ebF = l -> f.apply(l).withSchema(CollectionSchemaParam.newBuilder().withFieldTypes(fieldTypes).build());
final LongFunction<CreateCollectionParam.Builder> lastF = ebF;
return l -> lastF.apply(l).build();
@ -95,10 +95,9 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser<Cre
* Function to build the {@link FieldType}s for the {@link CreateCollectionParam}.
*
* @param fieldTypesData The static map of config data from the create collection request
* @param ebF
* @return a list of static field types
*/
private List<FieldType> buildFieldTypesStruct(Map<String, ParsedOp> fieldTypesData, LongFunction<CreateCollectionParam.Builder> ebF) {
private List<FieldType> buildFieldTypesStruct(Map<String, ParsedOp> fieldTypesData) {
List<FieldType> fieldTypes = new ArrayList<>();
fieldTypesData.forEach((name, fieldspec) -> {
FieldType.Builder builder = FieldType.newBuilder()

4
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/scheduling/TimeoutPredicateTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2024 nosqlbench
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -43,7 +43,7 @@ public class TimeoutPredicateTest {
assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.pending);
resultNow = wontMakeIt.test();
assertThat(resultNow.duration_ns()).isBetween(10*1_000_000L,50*1_000_000L);
assertThat(resultNow.duration_ns()).isBetween(10 * 1_000_000L, 50 * 1_000_000_000L);
assertThat(resultNow.value()).isFalse();
assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.pending);

8
nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTemplateVarTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022-2023 nosqlbench
* Copyright (c) 2022-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
@Test
public void testMultipleOccurencesOfSameTemplateVar() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars" }, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
@ -47,7 +47,7 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
@Test
public void testThatCLIOverridesWorkForTemplateVars() {
NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars", "tvar1=overridden" }, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars", "tvar1=overridden"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
@ -59,7 +59,7 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
@Test
public void testThatAdditionalCLIParamIsAdded() {
NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
assertThat(cmds).hasSize(2);

6
nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022-2023 nosqlbench
* Copyright (c) 2022-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,7 +33,7 @@ public class NBCLIScenarioPreprocessorTest {
@Test
public void providePathForScenario() {
NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts = new NBCLIOptions(new String[]{"example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds = opts.getCommands();
}
@ -188,7 +188,7 @@ public class NBCLIScenarioPreprocessorTest {
"tags", "block:\"schema.*\"",
"workload", "scenario_test"
));
NBCLIOptions opts1 = new NBCLIOptions(new String[]{"local/example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
NBCLIOptions opts1 = new NBCLIOptions(new String[]{"example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
List<Cmd> cmds1 = opts1.getCommands();
assertThat(cmds1.size()).isEqualTo(1);
assertThat(cmds1.get(0).getArgValueOrNull("cycles_test")).isNull();

15
nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios.yaml Normal file
View File

@ -0,0 +1,15 @@
# example_scenarios.yaml
scenarios:
default:
one: run cycles=3 alias=A driver=stdout
two: run cycles=5 alias=B driver=stdout
namedsteps:
one: run cycles=3 alias=A driver=stdout testparam1=testvalue1
two: run cycles=5 alias=B driver=stdout
bindings:
cycle: Identity()
name: NumberNameToCycle()
ops:
cycle: "cycle {cycle}\n"

10
nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml Normal file
View File

@ -0,0 +1,10 @@
# example_scenarios_templatevars.yaml
scenarios:
default:
first: run cycles=3 alias=A driver=stdout tvar1=replaced
second: run cycles=5 alias=B driver=stdout
bindings:
cycle: Identity()
name: NumberNameToCycle()
ops:
cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n"