Merge branch 'main' into environment_in_atfile

nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JSpace.java

@@ -18,6 +18,7 @@ package io.nosqlbench.adapter.neo4j;
 
 import io.nosqlbench.nb.api.config.standard.*;
 
+import io.nosqlbench.nb.api.errors.BasicError;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,6 +28,10 @@ import org.neo4j.driver.GraphDatabase;
 import org.neo4j.driver.*;
 import org.neo4j.driver.async.AsyncSession;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Optional;
 
 public class Neo4JSpace implements AutoCloseable {
@@ -48,42 +53,59 @@ public class Neo4JSpace implements AutoCloseable {
         this.sessionConfig = builder.build();
 
         String dbURI = cfg.get("db_uri");
+
         Optional<String> usernameOpt = cfg.getOptional("username");
+        Optional<String> userfileOpt = cfg.getOptional("userfile");
         Optional<String> passwordOpt = cfg.getOptional("password");
-        String username;
-        String password;
-        // user has supplied both username and password
-        if (usernameOpt.isPresent() && passwordOpt.isPresent()) {
+        Optional<String> passfileOpt = cfg.getOptional("passfile");
+
+        String username = null;
+        if (usernameOpt.isPresent()) {
             username = usernameOpt.get();
-            password = passwordOpt.get();
-            logger.info(this.space + ": Creating new Neo4J driver with [" +
-                "dbURI = " + dbURI +
-                ", username = " + username +
-                ", password = " + Neo4JAdapterUtils.maskDigits(password) +
-                "]"
-            );
+        } else if (userfileOpt.isPresent()) {
+            Path path = Paths.get(userfileOpt.get());
+            try {
+                username = Files.readAllLines(path).get(0);
+            } catch (IOException e) {
+                String error = "Error while reading username from file:" + path;
+                logger.error(error, e);
+                throw new RuntimeException(e);
+            }
+        }
+
+        String password = null;
+        if (username != null) {
+
+            if (passwordOpt.isPresent()) {
+                password = passwordOpt.get();
+            } else if (passfileOpt.isPresent()) {
+                Path path = Paths.get(passfileOpt.get());
+                try {
+                    password = Files.readAllLines(path).get(0);
+                } catch (IOException e) {
+                    String error = "Error while reading password from file:" + path;
+                    logger.error(error, e);
+                    throw new RuntimeException(e);
+                }
+            } else {
+                String error = "username is present, but neither password nor passfile are defined.";
+                logger.error(error);
+                throw new RuntimeException(error);
+            }
+        }
+
+        if ((username == null) != (password == null)) {
+            throw new BasicError("You must provide both username and password, or neither, with either " +
+                "username|userfile and password|passfile options");
+        }
+        if (username != null) {
             return GraphDatabase.driver(dbURI, AuthTokens.basic(username, password));
+        } else {
+
         }
-        // user has only supplied username
-        else if (usernameOpt.isPresent()) {
-            String error = "username is present, but password is not defined.";
-            logger.error(error);
-            throw new RuntimeException(error);
-        }
-        // user has only supplied password
-        else if (passwordOpt.isPresent()) {
-            String error = "password is present, but username is not defined.";
-            logger.error(error);
-            throw new RuntimeException(error);
-        }
-        // user has supplied neither
-        else {
-            logger.info(this.space + ": Creating new Neo4J driver with [" +
-                "dbURI = " + dbURI +
-                "]"
-            );
-            return GraphDatabase.driver(dbURI);
-        }
+
+        // user has supplied both username and password
+        return GraphDatabase.driver(dbURI);
     }
 
     public static NBConfigModel getConfigModel() {
@@ -92,6 +114,8 @@ public class Neo4JSpace implements AutoCloseable {
             .add(Param.optional("username", String.class))
             .add(Param.optional("password", String.class))
             .add(Param.optional("database", String.class))
+            .add(Param.optional("userfile", String.class))
+            .add(Param.optional("passfile", String.class))
             .asReadOnly();
     }
 
@@ -100,7 +124,7 @@ public class Neo4JSpace implements AutoCloseable {
     }
 
     public AsyncSession newAsyncSession() {
-        return driver.session(AsyncSession.class,sessionConfig);
+        return driver.session(AsyncSession.class, sessionConfig);
     }
 
     public Session newSession() {
@@ -109,7 +133,7 @@ public class Neo4JSpace implements AutoCloseable {
 
     @Override
     public void close() throws Exception {
-        if (driver != null){
+        if (driver != null) {
             driver.close();
         }
     }

nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java

@@ -22,6 +22,7 @@ import io.nosqlbench.nb.api.nbio.NBPathsAPI;
 import io.nosqlbench.nb.api.system.NBEnvironment;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 import org.snakeyaml.engine.v2.api.Load;
 import org.snakeyaml.engine.v2.api.LoadSettings;
 
@@ -50,6 +51,12 @@ public class NBAtFile {
      *     <LI>{@code >-- } asserts each value starts with global option syntax (--)</LI>
      * </UL>
      *
+     * <P>Files can be included recursively using a format like <PRE>{@code
+     * - include:${DIR}/somefile.yaml
+     * }</PRE></P>
+     *
+     *  Standard formatting specifiers above should work in this mode as well.
+     *
      * @param processInPlace The linked list which is statefully modified. If you need
      *                       an unmodified copy, then this is the responsibility of the caller.
      * @return An updated list with all values expanded and injected
@@ -60,10 +67,10 @@ public class NBAtFile {
         ListIterator<String> iter = processInPlace.listIterator();
         while (iter.hasNext()) {
             String spec = iter.next();
-            if (spec.startsWith("@")) {
+            if (spec.startsWith("@") || spec.startsWith("include=")|| spec.startsWith("include:")) {
                 iter.previous();
                 iter.remove();
-                LinkedList<String> spliceIn = includeAt(spec);
+                LinkedList<String> spliceIn = includeAt(spec.replaceFirst("include=","@").replaceFirst("include:","@"));
                 for (String s : spliceIn) {
                     iter.add(s);
                 }
@@ -90,6 +97,22 @@ public class NBAtFile {
      * @return The linked list of arguments which is to be spliced into the caller's command list
      */
     public static LinkedList<String> includeAt(String spec) {
+        LinkedList<String> toInclude = doInclude(spec);
+        boolean recurse = false;
+        for (String s : toInclude) {
+            if (s.startsWith("include=")||s.startsWith("include:")) {
+                recurse=true;
+                break;
+            }
+        }
+        if (recurse) {
+            toInclude=includeAt(toInclude);
+        }
+        return toInclude;
+
+    }
+
+    private static @NotNull LinkedList<String> doInclude(String spec) {
         Matcher matcher = includePattern.matcher(spec);
         if (matcher.matches()) {
             String filepathSpec = matcher.group("filepath");
@@ -136,7 +159,6 @@ public class NBAtFile {
         } else {
             throw new RuntimeException("Unable to match at-file specifier: " + spec + " to pattern '" + includePattern.pattern() + "'");
         }
-
     }
 
     private static LinkedList<String> interposePath(LinkedList<String> formatted, Path atPath) {

nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java

@@ -79,4 +79,16 @@ class NBAtFileTest {
         assertThat(strings).containsExactly("--option1", "--option2=value2", "--option3=value3", "--option4=value4");
     }
 
+    @Test
+    public void testAtfileSimpleRecursion() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_recursion.yaml");
+        assertThat(strings).containsExactly("arg1","arg1","arg2","arg3","arg3");
+    }
+
+    @Test
+    public void testAtfileDoubleRecursion() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/double_recursion.yaml");
+        assertThat(strings).containsExactly("arg1","arg1","arg1","arg2","arg3","arg3","arg3","deepval");
+    }
+
 }

nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml

@@ -0,0 +1 @@
+- deepval

nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml

@@ -0,0 +1,4 @@
+- arg1
+- include:${DIR}/simple_recursion.yaml
+- arg3
+- include:${DIR}/deeper/deeper_recursion.yaml

nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml

@@ -0,0 +1,3 @@
+- arg1
+- include:${DIR}/simple_list.yaml
+- arg3