From 5190bad4430490b04dfa0d493041c1646b3be8a1 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 10 Jul 2024 11:30:41 -0500 Subject: [PATCH 1/3] add userfile and passfile to neo param model --- .../nosqlbench/adapter/neo4j/Neo4JSpace.java | 90 ++++++++++++------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JSpace.java b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JSpace.java index 5f81a917d..d046b6f85 100644 --- a/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JSpace.java +++ b/nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JSpace.java @@ -18,6 +18,7 @@ package io.nosqlbench.adapter.neo4j; import io.nosqlbench.nb.api.config.standard.*; +import io.nosqlbench.nb.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -27,6 +28,10 @@ import org.neo4j.driver.GraphDatabase; import org.neo4j.driver.*; import org.neo4j.driver.async.AsyncSession; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Optional; public class Neo4JSpace implements AutoCloseable { @@ -48,42 +53,59 @@ public class Neo4JSpace implements AutoCloseable { this.sessionConfig = builder.build(); String dbURI = cfg.get("db_uri"); + Optional usernameOpt = cfg.getOptional("username"); + Optional userfileOpt = cfg.getOptional("userfile"); Optional passwordOpt = cfg.getOptional("password"); - String username; - String password; - // user has supplied both username and password - if (usernameOpt.isPresent() && passwordOpt.isPresent()) { + Optional passfileOpt = cfg.getOptional("passfile"); + + String username = null; + if (usernameOpt.isPresent()) { username = usernameOpt.get(); - password = passwordOpt.get(); - logger.info(this.space + ": Creating new Neo4J driver with [" + - "dbURI = " + dbURI + - ", username = " + username + - ", password = " + Neo4JAdapterUtils.maskDigits(password) + - "]" - ); + } else if (userfileOpt.isPresent()) { + Path path = Paths.get(userfileOpt.get()); + try { + username = Files.readAllLines(path).get(0); + } catch (IOException e) { + String error = "Error while reading username from file:" + path; + logger.error(error, e); + throw new RuntimeException(e); + } + } + + String password = null; + if (username != null) { + + if (passwordOpt.isPresent()) { + password = passwordOpt.get(); + } else if (passfileOpt.isPresent()) { + Path path = Paths.get(passfileOpt.get()); + try { + password = Files.readAllLines(path).get(0); + } catch (IOException e) { + String error = "Error while reading password from file:" + path; + logger.error(error, e); + throw new RuntimeException(e); + } + } else { + String error = "username is present, but neither password nor passfile are defined."; + logger.error(error); + throw new RuntimeException(error); + } + } + + if ((username == null) != (password == null)) { + throw new BasicError("You must provide both username and password, or neither, with either " + + "username|userfile and password|passfile options"); + } + if (username != null) { return GraphDatabase.driver(dbURI, AuthTokens.basic(username, password)); + } else { + } - // user has only supplied username - else if (usernameOpt.isPresent()) { - String error = "username is present, but password is not defined."; - logger.error(error); - throw new RuntimeException(error); - } - // user has only supplied password - else if (passwordOpt.isPresent()) { - String error = "password is present, but username is not defined."; - logger.error(error); - throw new RuntimeException(error); - } - // user has supplied neither - else { - logger.info(this.space + ": Creating new Neo4J driver with [" + - "dbURI = " + dbURI + - "]" - ); - return GraphDatabase.driver(dbURI); - } + + // user has supplied both username and password + return GraphDatabase.driver(dbURI); } public static NBConfigModel getConfigModel() { @@ -92,6 +114,8 @@ public class Neo4JSpace implements AutoCloseable { .add(Param.optional("username", String.class)) .add(Param.optional("password", String.class)) .add(Param.optional("database", String.class)) + .add(Param.optional("userfile", String.class)) + .add(Param.optional("passfile", String.class)) .asReadOnly(); } @@ -100,7 +124,7 @@ public class Neo4JSpace implements AutoCloseable { } public AsyncSession newAsyncSession() { - return driver.session(AsyncSession.class,sessionConfig); + return driver.session(AsyncSession.class, sessionConfig); } public Session newSession() { @@ -109,7 +133,7 @@ public class Neo4JSpace implements AutoCloseable { @Override public void close() throws Exception { - if (driver != null){ + if (driver != null) { driver.close(); } } From 29e1dbe12280b427be4a5ad7a7a14ad125fc0046 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 10 Jul 2024 13:03:18 -0500 Subject: [PATCH 2/3] add basic recursion to atfiles --- .../engine/cli/atfiles/NBAtFile.java | 28 +++++++++++++++++-- .../engine/cli/atfiles/NBAtFileTest.java | 5 ++++ .../resources/atfiles/simple_recursion.yaml | 3 ++ 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java index f47ce4dfa..584481543 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java @@ -21,6 +21,7 @@ import io.nosqlbench.nb.api.nbio.NBIO; import io.nosqlbench.nb.api.nbio.NBPathsAPI; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.LoadSettings; @@ -49,6 +50,12 @@ public class NBAtFile { *
  • {@code >-- } asserts each value starts with global option syntax (--)
    • * * + *

    Files can be included recursively using a format like 

    {@code
+     * - include:${DIR}/somefile.yaml
+     * }

    + * + * Standard formatting specifiers above should work in this mode as well. + * * @param processInPlace The linked list which is statefully modified. If you need * an unmodified copy, then this is the responsibility of the caller. * @return An updated list with all values expanded and injected @@ -59,10 +66,10 @@ public class NBAtFile { ListIterator iter = processInPlace.listIterator(); while (iter.hasNext()) { String spec = iter.next(); - if (spec.startsWith("@")) { + if (spec.startsWith("@") || spec.startsWith("include=")|| spec.startsWith("include:")) { iter.previous(); iter.remove(); - LinkedList spliceIn = includeAt(spec); + LinkedList spliceIn = includeAt(spec.replaceFirst("include=","@").replaceFirst("include:","@")); for (String s : spliceIn) { iter.add(s); } @@ -89,6 +96,22 @@ public class NBAtFile { * @return The linked list of arguments which is to be spliced into the caller's command list */ public static LinkedList includeAt(String spec) { + LinkedList toInclude = doInclude(spec); + boolean recurse = false; + for (String s : toInclude) { + if (s.startsWith("include=")||s.startsWith("include:")) { + recurse=true; + break; + } + } + if (recurse) { + toInclude=includeAt(toInclude); + } + return toInclude; + + } + + private static @NotNull LinkedList doInclude(String spec) { Matcher matcher = includePattern.matcher(spec); if (matcher.matches()) { String filepathSpec = matcher.group("filepath"); @@ -135,7 +158,6 @@ public class NBAtFile { } else { throw new RuntimeException("Unable to match at-file specifier: " + spec + " to pattern '" + includePattern.pattern() + "'"); } - } private static LinkedList interposePath(LinkedList formatted, Path atPath) { diff --git a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java index c42744096..fc53effa8 100644 --- a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java +++ b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java @@ -79,4 +79,9 @@ class NBAtFileTest { assertThat(strings).containsExactly("--option1", "--option2=value2", "--option3=value3", "--option4=value4"); } + @Test + public void testAtfileRecursion() { + LinkedList strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_recursion.yaml"); + assertThat(strings).containsExactly("arg1","arg1","arg2","arg3","arg3"); + } } diff --git a/nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml b/nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml new file mode 100644 index 000000000..08adde1c2 --- /dev/null +++ b/nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml @@ -0,0 +1,3 @@ +- arg1 +- include:${DIR}/simple_list.yaml +- arg3 From 60bf8bd6315299fdabd0bb17d074967283b2ae63 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 10 Jul 2024 13:33:30 -0500 Subject: [PATCH 3/3] add multi-layer recursion and path-relative recursion tests --- .../io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java | 9 ++++++++- .../test/resources/atfiles/deeper/deeper_recursion.yaml | 1 + .../src/test/resources/atfiles/double_recursion.yaml | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml create mode 100644 nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml diff --git a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java index fc53effa8..be2b6e08e 100644 --- a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java +++ b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java @@ -80,8 +80,15 @@ class NBAtFileTest { } @Test - public void testAtfileRecursion() { + public void testAtfileSimpleRecursion() { LinkedList strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_recursion.yaml"); assertThat(strings).containsExactly("arg1","arg1","arg2","arg3","arg3"); } + + @Test + public void testAtfileDoubleRecursion() { + LinkedList strings = NBAtFile.includeAt("@src/test/resources/atfiles/double_recursion.yaml"); + assertThat(strings).containsExactly("arg1","arg1","arg1","arg2","arg3","arg3","arg3","deepval"); + } + } diff --git a/nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml b/nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml new file mode 100644 index 000000000..d01cd1602 --- /dev/null +++ b/nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml @@ -0,0 +1 @@ +- deepval diff --git a/nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml b/nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml new file mode 100644 index 000000000..168042b8a --- /dev/null +++ b/nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml @@ -0,0 +1,4 @@ +- arg1 +- include:${DIR}/simple_recursion.yaml +- arg3 +- include:${DIR}/deeper/deeper_recursion.yaml