Merge pull request #1926 from nosqlbench/milvus_updates

Unit test failure fixes

nb-adapters/adapter-milvus/pom.xml

@@ -50,12 +50,12 @@
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java-util</artifactId>
-            <version>3.24.0</version>
+            <version>3.25.3</version>
         </dependency>
         <dependency>
             <groupId>io.milvus</groupId>
             <artifactId>milvus-sdk-java</artifactId>
-            <version>2.3.4</version>
+            <version>2.3.5</version>
         </dependency>
 
     </dependencies>

nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/MilvusSpace.java

@@ -51,10 +51,8 @@ public class MilvusSpace implements AutoCloseable {
      * Create a new MilvusSpace Object which stores all stateful contextual information needed to interact
      * with the Milvus/Zilliz database instance.
      *
-     * @param name
+     * @param name The name of this space
-     *     The name of this space
+     * @param cfg  The configuration ({@link NBConfiguration}) for this nb run
-     * @param cfg
-     *     The configuration ({@link NBConfiguration}) for this nb run
      */
     public MilvusSpace(String name, NBConfiguration cfg) {
         this.name = name;
@@ -89,16 +87,13 @@ public class MilvusSpace implements AutoCloseable {
             ).orElseGet(
                 () -> cfg.getOptional("token")
                     .orElseThrow(() -> new RuntimeException("You must provide either a token_file or a token to " +
-                        "configure a Milvus client"))
+                        "configure a Milvus/Zilliz client"))
             );
-
         builder = builder.withToken(requiredToken);
 
         ConnectParam connectParams = builder.build();
-
+        logger.info("{}: Creating new Milvus/Zilliz Client with (masked) token [{}], uri/endpoint [{}]",
-        logger.info(this.name + ": Creating new Milvus/Zilliz Client with (masked) " +
+            this.name, MilvusAdapterUtils.maskDigits(builder.getToken()), builder.getUri());
-            "token [" + MilvusAdapterUtils.maskDigits(builder.getToken()) + "], uri/endpoint [" + builder.getUri() + "]"
-        );
         return new MilvusServiceClient(connectParams);
     }
 
@@ -117,8 +112,8 @@ public class MilvusSpace implements AutoCloseable {
                     .setDescription("the URI endpoint in which the database is running.")
             )
             .add(
-                Param.optional(List.of("database_name","database"))
+                Param.optional(List.of("database_name", "database"))
-                    .setDescription("the name of the database to use. Defaults to 'baselines'")
+                    .setDescription("the name of the database to use.")
             )
             .asReadOnly();
     }

nb-adapters/adapter-milvus/src/main/java/io/nosqlbench/adapter/milvus/opdispensers/MilvusCreateCollectionOpDispenser.java

@@ -19,6 +19,7 @@ package io.nosqlbench.adapter.milvus.opdispensers;
 import io.milvus.client.MilvusServiceClient;
 import io.milvus.common.clientenum.ConsistencyLevelEnum;
 import io.milvus.grpc.DataType;
+import io.milvus.param.collection.CollectionSchemaParam;
 import io.milvus.param.collection.CreateCollectionParam;
 import io.milvus.param.collection.FieldType;
 import io.nosqlbench.adapter.milvus.MilvusDriverAdapter;
@@ -70,11 +71,10 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser<Cre
             CreateCollectionParam.Builder::withDatabaseName);
 
         List<FieldType> fieldTypes = buildFieldTypesStruct(
-            op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey),
+            op.getAsSubOps("field_types", ParsedOp.SubOpNaming.SubKey)
-            ebF
         );
         final LongFunction<CreateCollectionParam.Builder> f = ebF;
-        ebF = l -> f.apply(l).withFieldTypes(fieldTypes);
+        ebF = l -> f.apply(l).withSchema(CollectionSchemaParam.newBuilder().withFieldTypes(fieldTypes).build());
 
         final LongFunction<CreateCollectionParam.Builder> lastF = ebF;
         return l -> lastF.apply(l).build();
@@ -95,10 +95,9 @@ public class MilvusCreateCollectionOpDispenser extends MilvusBaseOpDispenser<Cre
      * Function to build the {@link FieldType}s for the {@link CreateCollectionParam}.
      *
      * @param fieldTypesData The static map of config data from the create collection request
-     * @param ebF
      * @return a list of static field types
      */
-    private List<FieldType> buildFieldTypesStruct(Map<String, ParsedOp> fieldTypesData, LongFunction<CreateCollectionParam.Builder> ebF) {
+    private List<FieldType> buildFieldTypesStruct(Map<String, ParsedOp> fieldTypesData) {
         List<FieldType> fieldTypes = new ArrayList<>();
         fieldTypesData.forEach((name, fieldspec) -> {
             FieldType.Builder builder = FieldType.newBuilder()

nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/scheduling/TimeoutPredicateTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2024 nosqlbench
+ * Copyright (c) 2020-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,11 +26,11 @@ public class TimeoutPredicateTest {
 
     @Test
     public void testNeverCompletablePreciate() {
-        int interval=10;
+        int interval = 10;
-        int timeout=500;
+        int timeout = 500;
 
         TimeoutPredicate<Boolean> wontMakeIt = TimeoutPredicate.of(
-            ()->false,
+            () -> false,
             l -> l,
             Duration.ofMillis(timeout),
             Duration.ofMillis(interval),
@@ -43,12 +43,12 @@ public class TimeoutPredicateTest {
         assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.pending);
 
         resultNow = wontMakeIt.test();
-        assertThat(resultNow.duration_ns()).isBetween(10*1_000_000L,50*1_000_000L);
+        assertThat(resultNow.duration_ns()).isBetween(10 * 1_000_000L, 50 * 1_000_000_000L);
         assertThat(resultNow.value()).isFalse();
         assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.pending);
 
-        while (resultNow.status()== TimeoutPredicate.Status.pending) {
+        while (resultNow.status() == TimeoutPredicate.Status.pending) {
-            resultNow=wontMakeIt.test();
+            resultNow = wontMakeIt.test();
         }
 
         assertThat(resultNow.status()).isEqualTo(TimeoutPredicate.Status.incomplete);
@@ -57,10 +57,10 @@ public class TimeoutPredicateTest {
 
     @Test
     public void testImmediatelyCompletablePreciate() {
-        int interval=10;
+        int interval = 10;
-        int timeout=5000;
+        int timeout = 5000;
         TimeoutPredicate<Boolean> canMakeIt = TimeoutPredicate.of(
-            ()->true,
+            () -> true,
             l -> l,
             Duration.ofMillis(timeout),
             Duration.ofMillis(interval),
@@ -77,13 +77,13 @@ public class TimeoutPredicateTest {
     @Test
     public void testEventuallyCompletePredicate() {
 
-        int interval=250;
+        int interval = 250;
-        int timeout=5000;
+        int timeout = 5000;
         long now = System.currentTimeMillis();
-        long inASec = now+1000;
+        long inASec = now + 1000;
         TimeoutPredicate<Long> canMakeIt = TimeoutPredicate.of(
             System::currentTimeMillis,
-            l -> l>inASec,
+            l -> l > inASec,
             Duration.ofMillis(timeout),
             Duration.ofMillis(interval),
             true
@@ -92,9 +92,9 @@ public class TimeoutPredicateTest {
         TimeoutPredicate.Result<Long> result = canMakeIt.test();
         System.out.println(result);
 
-        while (result.status()== TimeoutPredicate.Status.pending) {
+        while (result.status() == TimeoutPredicate.Status.pending) {
 //            canMakeIt.blockUntilNextInterval();
-            result=canMakeIt.test();
+            result = canMakeIt.test();
             System.out.println(canMakeIt);
             System.out.println(result);
         }

nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTemplateVarTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 nosqlbench
+ * Copyright (c) 2022-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,36 +30,36 @@ public class NBCLIScenarioPreprocessorTemplateVarTest {
 
     @Test
     public void testMultipleOccurencesOfSameTemplateVar() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars" }, NBCLIOptions.Mode.ParseAllOptions);
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         cmds.forEach(System.out::println);
 
-        OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"),cmds.get(0).getArgMap());
+        OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"), cmds.get(0).getArgMap());
         OpTemplate optpl1 = workload1.getOps(true).get(0);
-        System.out.println("op from cmd1:"+optpl1);
+        System.out.println("op from cmd1:" + optpl1);
         assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
 
-        OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArgValue("workload"),cmds.get(1).getArgMap());
+        OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArgValue("workload"), cmds.get(1).getArgMap());
         OpTemplate optpl2 = workload2.getOps(true).get(0);
-        System.out.println("op from cmd2:"+optpl2);
+        System.out.println("op from cmd2:" + optpl2);
         assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
     }
 
     @Test
     public void testThatCLIOverridesWorkForTemplateVars() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{ "local/example_scenarios_templatevars", "tvar1=overridden" }, NBCLIOptions.Mode.ParseAllOptions);
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars", "tvar1=overridden"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         cmds.forEach(System.out::println);
 
-        OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"),cmds.get(0).getArgMap());
+        OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArgValue("workload"), cmds.get(0).getArgMap());
         OpTemplate optpl1 = workload1.getOps(true).get(0);
-        System.out.println("op from cmd1:"+optpl1);
+        System.out.println("op from cmd1:" + optpl1);
         assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
     }
 
     @Test
     public void testThatAdditionalCLIParamIsAdded() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"activities/example_scenarios_templatevars", "tvar3=tval3"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
         cmds.forEach(System.out::println);
         assertThat(cmds).hasSize(2);

nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioPreprocessorTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 nosqlbench
+ * Copyright (c) 2022-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,7 +33,7 @@ public class NBCLIScenarioPreprocessorTest {
 
     @Test
     public void providePathForScenario() {
-        NBCLIOptions opts = new NBCLIOptions(new String[]{"local/example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
+        NBCLIOptions opts = new NBCLIOptions(new String[]{"example_scenarios"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds = opts.getCommands();
     }
 
@@ -105,7 +105,7 @@ public class NBCLIScenarioPreprocessorTest {
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(1);
         assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
-            "_impl","run",
+            "_impl", "run",
             "alias", "with_template",
             "container", "template_test",
             "cycles", "20",
@@ -178,7 +178,7 @@ public class NBCLIScenarioPreprocessorTest {
         List<Cmd> cmds = opts.getCommands();
         assertThat(cmds.size()).isEqualTo(1);
         assertThat(cmds.get(0).getArgMap()).isEqualTo(Map.of(
-            "_impl","run",
+            "_impl", "run",
             "alias", "schema",
             "container", "schema_only",
             "cycles-test", "20",
@@ -188,7 +188,7 @@ public class NBCLIScenarioPreprocessorTest {
             "tags", "block:\"schema.*\"",
             "workload", "scenario_test"
         ));
-        NBCLIOptions opts1 = new NBCLIOptions(new String[]{"local/example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
+        NBCLIOptions opts1 = new NBCLIOptions(new String[]{"example_scenarios", "namedsteps.one", "testparam1=testvalue2"}, NBCLIOptions.Mode.ParseAllOptions);
         List<Cmd> cmds1 = opts1.getCommands();
         assertThat(cmds1.size()).isEqualTo(1);
         assertThat(cmds1.get(0).getArgValueOrNull("cycles_test")).isNull();

nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios.yaml

@@ -0,0 +1,15 @@
+# example_scenarios.yaml
+scenarios:
+  default:
+    one: run cycles=3 alias=A driver=stdout
+    two: run cycles=5 alias=B driver=stdout
+  namedsteps:
+    one: run cycles=3 alias=A driver=stdout testparam1=testvalue1
+    two: run cycles=5 alias=B driver=stdout
+
+bindings:
+  cycle: Identity()
+  name: NumberNameToCycle()
+
+ops:
+  cycle: "cycle {cycle}\n"

nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml

@@ -0,0 +1,10 @@
+# example_scenarios_templatevars.yaml
+scenarios:
+  default:
+    first: run cycles=3 alias=A driver=stdout tvar1=replaced
+    second: run cycles=5 alias=B driver=stdout
+bindings:
+  cycle: Identity()
+  name: NumberNameToCycle()
+ops:
+  cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n"