IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into snyk-upgrade-0ddc495dd7549416e8eda1fa8b67e430

Browse Source
This commit is contained in:
Jonathan Shook
2024-07-17 10:48:04 -05:00
committed by GitHub
parent ef4518f154 e8bbfbdf9b
commit 8ff7fafbb5
9 changed files with 143 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
mvn-defaults/pom.xml
View File

@@ -26,7 +26,7 @@
<properties>
<revision>5.21.1-SNAPSHOT</revision>
<revision>5.21.2-SNAPSHOT</revision>
<!-- Set this level to override the logging level for tests during build -->
<project.testlevel>INFO</project.testlevel>
<!-- Set this level to override the logging level for tests logging configuration during build -->

58
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java
View File

@@ -64,28 +64,10 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser<DataApiBaseOp,
List<Filter> orFilterList = new ArrayList<>();
for (Map<String,Object> filterFields : filters) {
switch ((String)filterFields.get("conjunction")) {
case "and" -> {
switch (filterFields.get("operator").toString()) {
case "lt" ->
andFilterList.add(Filters.lt(filterFields.get("field").toString(), (long) filterFields.get("value")));
case "gt" ->
andFilterList.add(Filters.gt(filterFields.get("field").toString(), (long) filterFields.get("value")));
case "eq" ->
andFilterList.add(Filters.eq(filterFields.get("field").toString(), filterFields.get("value")));
default -> logger.error(() -> "Operation " + filterFields.get("operator") + " not supported");
}
}
case "or" -> {
switch (filterFields.get("operator").toString()) {
case "lt" ->
orFilterList.add(Filters.lt(filterFields.get("field").toString(), (long) filterFields.get("value")));
case "gt" ->
orFilterList.add(Filters.gt(filterFields.get("field").toString(), (long) filterFields.get("value")));
case "eq" ->
orFilterList.add(Filters.eq(filterFields.get("field").toString(), filterFields.get("value")));
default -> logger.error(() -> "Operation " + filterFields.get("operator") + " not supported");
}
}
case "and" ->
addOperatorFilter(andFilterList, filterFields.get("operator").toString(), filterFields.get("field").toString(), filterFields.get("value"));
case "or" ->
addOperatorFilter(orFilterList, filterFields.get("operator").toString(), filterFields.get("field").toString(), filterFields.get("value"));
default -> logger.error(() -> "Conjunction " + filterFields.get("conjunction") + " not supported");
}
}
@@ -97,6 +79,38 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser<DataApiBaseOp,
return filter;
}
protected void addOperatorFilter(List<Filter> filtersList, String operator, String fieldName, Object fieldValue) {
switch (operator) {
case "all" ->
filtersList.add(Filters.all(fieldName, fieldValue));
case "eq" ->
filtersList.add(Filters.eq(fieldName, fieldValue));
case "exists" -> {
if (fieldValue != null) {
logger.warn(() -> "'exists' operator does not support value field");
}
filtersList.add(Filters.exists(fieldName));
}
case "gt" ->
filtersList.add(Filters.gt(fieldName, (long) fieldValue));
case "gte" ->
filtersList.add(Filters.gte(fieldName, (long) fieldValue));
case "hasSize" ->
filtersList.add(Filters.hasSize(fieldName, (int) fieldValue));
case "in" ->
filtersList.add(Filters.in(fieldName, fieldValue));
case "lt" ->
filtersList.add(Filters.lt(fieldName, (long) fieldValue));
case "lte" ->
filtersList.add(Filters.lte(fieldName, (long) fieldValue));
case "ne" ->
filtersList.add(Filters.ne(fieldName, fieldValue));
case "nin" ->
filtersList.add(Filters.nin(fieldName, fieldValue));
default -> logger.error(() -> "Operation '" + operator + "' not supported");
}
}
protected Update getUpdates(ParsedOp op, long l) {
Update update = new Update();
Optional<LongFunction<Map>> updatesFunction = op.getAsOptionalFunction("updates", Map.class);

82
nb-adapters/adapter-neo4j/src/main/java/io/nosqlbench/adapter/neo4j/Neo4JSpace.java
View File

@@ -18,6 +18,7 @@ package io.nosqlbench.adapter.neo4j;
import io.nosqlbench.nb.api.config.standard.*;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -27,6 +28,10 @@ import org.neo4j.driver.GraphDatabase;
import org.neo4j.driver.*;
import org.neo4j.driver.async.AsyncSession;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
public class Neo4JSpace implements AutoCloseable {
@@ -48,43 +53,60 @@ public class Neo4JSpace implements AutoCloseable {
this.sessionConfig = builder.build();
String dbURI = cfg.get("db_uri");
Optional<String> usernameOpt = cfg.getOptional("username");
Optional<String> userfileOpt = cfg.getOptional("userfile");
Optional<String> passwordOpt = cfg.getOptional("password");
String username;
String password;
// user has supplied both username and password
if (usernameOpt.isPresent() && passwordOpt.isPresent()) {
Optional<String> passfileOpt = cfg.getOptional("passfile");
String username = null;
if (usernameOpt.isPresent()) {
username = usernameOpt.get();
} else if (userfileOpt.isPresent()) {
Path path = Paths.get(userfileOpt.get());
try {
username = Files.readAllLines(path).get(0);
} catch (IOException e) {
String error = "Error while reading username from file:" + path;
logger.error(error, e);
throw new RuntimeException(e);
}
}
String password = null;
if (username != null) {
if (passwordOpt.isPresent()) {
password = passwordOpt.get();
logger.info(this.space + ": Creating new Neo4J driver with [" +
"dbURI = " + dbURI +
", username = " + username +
", password = " + Neo4JAdapterUtils.maskDigits(password) +
"]"
);
} else if (passfileOpt.isPresent()) {
Path path = Paths.get(passfileOpt.get());
try {
password = Files.readAllLines(path).get(0);
} catch (IOException e) {
String error = "Error while reading password from file:" + path;
logger.error(error, e);
throw new RuntimeException(e);
}
} else {
String error = "username is present, but neither password nor passfile are defined.";
logger.error(error);
throw new RuntimeException(error);
}
}
if ((username == null) != (password == null)) {
throw new BasicError("You must provide both username and password, or neither, with either " +
"username|userfile and password|passfile options");
}
if (username != null) {
return GraphDatabase.driver(dbURI, AuthTokens.basic(username, password));
} else {
}
// user has only supplied username
else if (usernameOpt.isPresent()) {
String error = "username is present, but password is not defined.";
logger.error(error);
throw new RuntimeException(error);
}
// user has only supplied password
else if (passwordOpt.isPresent()) {
String error = "password is present, but username is not defined.";
logger.error(error);
throw new RuntimeException(error);
}
// user has supplied neither
else {
logger.info(this.space + ": Creating new Neo4J driver with [" +
"dbURI = " + dbURI +
"]"
);
// user has supplied both username and password
return GraphDatabase.driver(dbURI);
}
}
public static NBConfigModel getConfigModel() {
return ConfigModel.of(Neo4JSpace.class)
@@ -92,6 +114,8 @@ public class Neo4JSpace implements AutoCloseable {
.add(Param.optional("username", String.class))
.add(Param.optional("password", String.class))
.add(Param.optional("database", String.class))
.add(Param.optional("userfile", String.class))
.add(Param.optional("passfile", String.class))
.asReadOnly();
}

2
nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/QdrantSpace.java
View File

@@ -72,7 +72,7 @@ public class QdrantSpace implements AutoCloseable {
boolean useTls = cfg.getOptional("use_tls").map(Boolean::parseBoolean).orElse(true);
var builder = QdrantGrpcClient.newBuilder(uri, grpcPort, useTls);
var Optional<requiredToken> = cfg.getOptional("token_file")
String requiredToken = cfg.getOptional("token_file")
.map(Paths::get)
.map(
tokenFilePath -> {

35
nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java
View File

@@ -21,6 +21,7 @@ import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.nbio.NBPathsAPI;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
@@ -49,6 +50,12 @@ public class NBAtFile {
* <LI>{@code >-- } asserts each value starts with global option syntax (--)</LI>
* </UL>
*
* <P>Files can be included recursively using a format like <PRE>{@code
* - include:${DIR}/somefile.yaml
* }</PRE></P>
*
* Standard formatting specifiers above should work in this mode as well.
*
* @param processInPlace The linked list which is statefully modified. If you need
* an unmodified copy, then this is the responsibility of the caller.
* @return An updated list with all values expanded and injected
@@ -59,10 +66,10 @@ public class NBAtFile {
ListIterator<String> iter = processInPlace.listIterator();
while (iter.hasNext()) {
String spec = iter.next();
if (spec.startsWith("@")) {
if (spec.startsWith("@") || spec.startsWith("include=")|| spec.startsWith("include:")) {
iter.previous();
iter.remove();
LinkedList<String> spliceIn = includeAt(spec);
LinkedList<String> spliceIn = includeAt(spec.replaceFirst("include=","@").replaceFirst("include:","@"));
for (String s : spliceIn) {
iter.add(s);
}
@@ -89,6 +96,22 @@ public class NBAtFile {
* @return The linked list of arguments which is to be spliced into the caller's command list
*/
public static LinkedList<String> includeAt(String spec) {
LinkedList<String> toInclude = doInclude(spec);
boolean recurse = false;
for (String s : toInclude) {
if (s.startsWith("include=")||s.startsWith("include:")) {
recurse=true;
break;
}
}
if (recurse) {
toInclude=includeAt(toInclude);
}
return toInclude;
}
private static @NotNull LinkedList<String> doInclude(String spec) {
Matcher matcher = includePattern.matcher(spec);
if (matcher.matches()) {
String filepathSpec = matcher.group("filepath");
@@ -96,12 +119,11 @@ public class NBAtFile {
String formatSpec = matcher.group("formatter");
String[] datapath = (dataPathSpec!=null && !dataPathSpec.isBlank()) ? dataPathSpec.split("(/|\\.)") : new String[] {};
String[] parts = filepathSpec.split("\\.",2);
if (parts.length==2 && !parts[1].toLowerCase().matches("yaml")) {
String filename = Path.of(filepathSpec).getFileName().toString();
if (filename.contains(".") && !(filename.toLowerCase().endsWith("yaml"))) {
throw new RuntimeException("Only the yaml format and extension is supported for at-files." +
" You specified " + parts[1]);
" You specified " + filepathSpec);
}
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml");
Path atPath = Path.of(filepathSpec);
@@ -135,7 +157,6 @@ public class NBAtFile {
} else {
throw new RuntimeException("Unable to match at-file specifier: " + spec + " to pattern '" + includePattern.pattern() + "'");
}
}
private static LinkedList<String> interposePath(LinkedList<String> formatted, Path atPath) {

12
nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java
View File

@@ -79,4 +79,16 @@ class NBAtFileTest {
assertThat(strings).containsExactly("--option1", "--option2=value2", "--option3=value3", "--option4=value4");
}
@Test
public void testAtfileSimpleRecursion() {
LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_recursion.yaml");
assertThat(strings).containsExactly("arg1","arg1","arg2","arg3","arg3");
}
@Test
public void testAtfileDoubleRecursion() {
LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/double_recursion.yaml");
assertThat(strings).containsExactly("arg1","arg1","arg1","arg2","arg3","arg3","arg3","deepval");
}
}

1
nb-engine/nb-engine-cli/src/test/resources/atfiles/deeper/deeper_recursion.yaml Normal file
View File

@@ -0,0 +1 @@
- deepval

4
nb-engine/nb-engine-cli/src/test/resources/atfiles/double_recursion.yaml Normal file
View File

@@ -0,0 +1,4 @@
- arg1
- include:${DIR}/simple_recursion.yaml
- arg3
- include:${DIR}/deeper/deeper_recursion.yaml

3
nb-engine/nb-engine-cli/src/test/resources/atfiles/simple_recursion.yaml Normal file
View File

@@ -0,0 +1,3 @@
- arg1
- include:${DIR}/simple_list.yaml
- arg3