Merge branch 'main' into environment_in_atfile

nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java

@@ -22,6 +22,7 @@ import io.nosqlbench.nb.api.nbio.NBPathsAPI;
 import io.nosqlbench.nb.api.system.NBEnvironment;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 import org.snakeyaml.engine.v2.api.Load;
 import org.snakeyaml.engine.v2.api.LoadSettings;
 
@@ -50,6 +51,12 @@ public class NBAtFile {
      *     <LI>{@code >-- } asserts each value starts with global option syntax (--)</LI>
      * </UL>
      *
+     * <P>Files can be included recursively using a format like <PRE>{@code
+     * - include:${DIR}/somefile.yaml
+     * }</PRE></P>
+     *
+     *  Standard formatting specifiers above should work in this mode as well.
+     *
      * @param processInPlace The linked list which is statefully modified. If you need
      *                       an unmodified copy, then this is the responsibility of the caller.
      * @return An updated list with all values expanded and injected
@@ -60,10 +67,10 @@ public class NBAtFile {
         ListIterator<String> iter = processInPlace.listIterator();
         while (iter.hasNext()) {
             String spec = iter.next();
-            if (spec.startsWith("@")) {
+            if (spec.startsWith("@") || spec.startsWith("include=")|| spec.startsWith("include:")) {
                 iter.previous();
                 iter.remove();
-                LinkedList<String> spliceIn = includeAt(spec);
+                LinkedList<String> spliceIn = includeAt(spec.replaceFirst("include=","@").replaceFirst("include:","@"));
                 for (String s : spliceIn) {
                     iter.add(s);
                 }
@@ -90,6 +97,22 @@ public class NBAtFile {
      * @return The linked list of arguments which is to be spliced into the caller's command list
      */
     public static LinkedList<String> includeAt(String spec) {
+        LinkedList<String> toInclude = doInclude(spec);
+        boolean recurse = false;
+        for (String s : toInclude) {
+            if (s.startsWith("include=")||s.startsWith("include:")) {
+                recurse=true;
+                break;
+            }
+        }
+        if (recurse) {
+            toInclude=includeAt(toInclude);
+        }
+        return toInclude;
+
+    }
+
+    private static @NotNull LinkedList<String> doInclude(String spec) {
         Matcher matcher = includePattern.matcher(spec);
         if (matcher.matches()) {
             String filepathSpec = matcher.group("filepath");
@@ -136,7 +159,6 @@ public class NBAtFile {
         } else {
             throw new RuntimeException("Unable to match at-file specifier: " + spec + " to pattern '" + includePattern.pattern() + "'");
         }
-
     }
 
     private static LinkedList<String> interposePath(LinkedList<String> formatted, Path atPath) {

nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java

@@ -79,4 +79,16 @@ class NBAtFileTest {
         assertThat(strings).containsExactly("--option1", "--option2=value2", "--option3=value3", "--option4=value4");
     }
 
+    @Test
+    public void testAtfileSimpleRecursion() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_recursion.yaml");
+        assertThat(strings).containsExactly("arg1","arg1","arg2","arg3","arg3");
+    }
+
+    @Test
+    public void testAtfileDoubleRecursion() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/double_recursion.yaml");
+        assertThat(strings).containsExactly("arg1","arg1","arg1","arg2","arg3","arg3","arg3","deepval");
+    }
+
 }

nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml

@@ -0,0 +1 @@
+- deepval

nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml

@@ -0,0 +1,4 @@
+- arg1
+- include:${DIR}/simple_recursion.yaml
+- arg3
+- include:${DIR}/deeper/deeper_recursion.yaml

nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml

@@ -0,0 +1,3 @@
+- arg1
+- include:${DIR}/simple_list.yaml
+- arg3