IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into snyk-upgrade-8bf908fcba73a5b4b98f8913dae7c67b

Browse Source
This commit is contained in:
Jonathan Shook
2024-01-16 19:33:29 -06:00
committed by GitHub
parent fff21d03dc 8130b3c547
commit b71131b99a
23 changed files with 220 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java
View File

@@ -37,6 +37,7 @@ import org.apache.logging.log4j.Logger;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -97,8 +98,18 @@ public class Cqld4Space implements AutoCloseable {
// int port = cfg.getOptional(int.class, "port").orElse(9042);
Optional<String> scb = cfg.getOptional(String.class, "secureconnectbundle", "scb");
scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream))
.map(builder::withCloudSecureConnectBundle);
if (scb.isPresent()) {
Optional<InputStream> stream =
scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream));
if (stream.isPresent()) {
stream.map(builder::withCloudSecureConnectBundle);
} else {
String error = String.format("Unable to load Secure Connect Bundle from path %s", scb.get());
logger.error(error);
throw new RuntimeException(error);
}
}
Optional<List<InetSocketAddress>> contactPointsOption = cfg
.getOptional("host", "hosts")

2
adapter-dynamodb/pom.xml
View File

@@ -43,7 +43,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
<version>1.12.622</version>
<version>1.12.635</version>
</dependency>
</dependencies>

2
adapter-http/pom.xml
View File

@@ -72,7 +72,7 @@
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-core</artifactId>
<version>2.2.19</version>
<version>2.2.20</version>
</dependency>
</dependencies>

2
adapter-pulsar/pom.xml
View File

@@ -34,7 +34,7 @@
</description>
<properties>
<pulsar.version>3.1.1</pulsar.version>
<pulsar.version>3.1.2</pulsar.version>
</properties>
<dependencies>

4
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java
View File

@@ -43,7 +43,7 @@ public class OpDef extends OpTemplate {
@Override
public String getName() {
return block.getName() + "__" + rawOpDef.getName();
return rawOpDef.getName();
}
@Override
@@ -100,7 +100,7 @@ public class OpDef extends OpTemplate {
private LinkedHashMap<String, String> composeTags() {
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags()));
tagsWithName.put("block",block.getName());
tagsWithName.put("name",getName());
tagsWithName.put("name",this.rawOpDef.getName());
tagsWithName.put("op",this.rawOpDef.getName());
return tagsWithName;
}

24
adapters-api/src/main/resources/workload_definition/02_workload_structure.md
View File

@@ -375,24 +375,24 @@ blocks:
[
{
"name": "namedblock1__op1",
"name": "op1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "namedblock1__op1",
"name": "op1",
"block": "namedblock1",
"op": "op1"
}
},
{
"name": "namedblock1__op2",
"name": "op2",
"op": {
"stmt": "insert into bar.table (a,b,c) values (1,2,3);",
"type": "batch"
},
"tags": {
"name": "namedblock1__op2",
"name": "op2",
"block": "namedblock1",
"op": "op2"
}
@@ -450,35 +450,35 @@ blocks:
[
{
"name": "block1__op1",
"name": "op1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block1__op1",
"name": "op1",
"block": "block1",
"op": "op1"
}
},
{
"name": "block1__op2",
"name": "op2",
"op": {
"stmt": "insert into bar.table (a,b,c) values (1,2,3);",
"type": "batch"
},
"tags": {
"name": "block1__op2",
"name": "op2",
"block": "block1",
"op": "op2"
}
},
{
"name": "this_is_block_2__op3",
"name": "op3",
"op": {
"stmt": "select * from foo.table;"
},
"tags": {
"name": "this_is_block_2__op3",
"name": "op3",
"block": "this_is_block_2",
"op": "op3"
}
@@ -523,12 +523,12 @@ blocks:
[
{
"name": "myblock__stmt1",
"name": "stmt1",
"op": {
"stmt": "test op"
},
"tags": {
"name": "myblock__stmt1",
"name": "stmt1",
"block": "myblock",
"op": "stmt1"
}

32
adapters-api/src/main/resources/workload_definition/04_op_template_basics.md
View File

@@ -34,12 +34,12 @@ op: select * from bar.table;
[
{
"name": "block0__stmt1",
"name": "stmt1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block0__stmt1",
"name": "stmt1",
"block": "block0",
"op": "stmt1"
}
@@ -73,12 +73,12 @@ ops:
[
{
"name": "block0__stmt1",
"name": "stmt1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block0__stmt1",
"name": "stmt1",
"block": "block0",
"op": "stmt1"
}
@@ -117,12 +117,12 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
}
@@ -160,12 +160,12 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
}
@@ -199,12 +199,12 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
}
@@ -241,12 +241,12 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"stmt": "select * from bar.table;"
},
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
}
@@ -315,7 +315,7 @@ ops:
"binding1": "NumberNameToString();"
},
"description": "This is just an example operation",
"name": "block0__special-op-name",
"name": "special-op-name",
"op": {
"stmt": "select * from ks1.tb1;"
},
@@ -324,7 +324,7 @@ ops:
},
"tags": {
"block": "block0",
"name": "block0__special-op-name",
"name": "special-op-name",
"op": "special-op-name"
}
}
@@ -414,7 +414,7 @@ blocks:
"bindings": {
"binding1": "NumberNameToString();"
},
"name": "block_named_fred__special-op-name",
"name": "special-op-name",
"op": {
"stmt": "select * from ks1.tb1;"
},
@@ -424,7 +424,7 @@ blocks:
"tags": {
"block": "block_named_fred",
"docleveltag": "is-tagging-everything",
"name": "block_named_fred__special-op-name",
"name": "special-op-name",
"op": "special-op-name"
}
}

36
adapters-api/src/main/resources/workload_definition/05_op_template_payloads.md
View File

@@ -39,14 +39,14 @@ ops: "cycle number '{{NumberNameToString}}'"
[
{
"tags": {
"name": "block0__stmt1",
"name": "stmt1",
"block": "block0",
"op": "stmt1"
},
"op": {
"stmt": "cycle number '{{NumberNameToString}}'"
},
"name": "block0__stmt1"
"name": "stmt1"
}
]
```
@@ -83,25 +83,25 @@ ops:
[
{
"tags": {
"name": "block0__stmt1",
"name": "stmt1",
"block": "block0",
"op": "stmt1"
},
"op": {
"stmt": "even cycle '{{NumberNameToString}}'"
},
"name": "block0__stmt1"
"name": "stmt1"
},
{
"tags": {
"name": "block0__stmt2",
"name": "stmt2",
"block": "block0",
"op": "stmt2"
},
"op": {
"stmt": "odd cycle '{{NumberNameToString}}'"
},
"name": "block0__stmt2"
"name": "stmt2"
}
]
```
@@ -138,25 +138,25 @@ ops:
[
{
"tags": {
"name": "block0__myop1",
"name": "myop1",
"block": "block0",
"op": "myop1"
},
"op": {
"stmt": "even cycle '{{NumberNameToString}}'"
},
"name": "block0__myop1"
"name": "myop1"
},
{
"tags": {
"name": "block0__myop2",
"name": "myop2",
"block": "block0",
"op": "myop2"
},
"op": {
"stmt": "odd cycle '{{NumberNameToString}}'"
},
"name": "block0__myop2"
"name": "myop2"
}
]
```
@@ -198,7 +198,7 @@ ops:
[
{
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
},
@@ -206,7 +206,7 @@ ops:
"opfield1": "opvalue1",
"opfield2": "opvalue2"
},
"name": "block0__op1"
"name": "op1"
}
]
```
@@ -250,7 +250,7 @@ ops:
[
{
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
},
@@ -260,7 +260,7 @@ ops:
"opvalue2"
]
},
"name": "block0__op1"
"name": "op1"
}
]
```
@@ -310,7 +310,7 @@ ops:
[
{
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
},
@@ -324,7 +324,7 @@ ops:
"by_session_len/@1h"
]
},
"name": "block0__op1"
"name": "op1"
}
]
@@ -384,7 +384,7 @@ ops:
"user_index": "Mod(1000L); ToString();"
},
"tags": {
"name": "block0__op1",
"name": "op1",
"block": "block0",
"op": "op1"
},
@@ -398,7 +398,7 @@ ops:
"by_session_len/@1h"
]
},
"name": "block0__op1"
"name": "op1"
}
]
```

48
adapters-api/src/main/resources/workload_definition/06_op_template_variations.md
View File

@@ -42,13 +42,13 @@ ops:
[
{
"name": "block0__special-op-name",
"name": "special-op-name",
"op": {
"stmt": "select * from ks1.tb1;"
},
"tags": {
"block": "block0",
"name": "block0__special-op-name",
"name": "special-op-name",
"op": "special-op-name"
}
}
@@ -94,14 +94,14 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"field1": "select * from ks1.tb1;",
"field2": "field 2 value"
},
"tags": {
"block": "block0",
"name": "block0__op1",
"name": "op1",
"op": "op1"
}
}
@@ -141,14 +141,14 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"field1": "select * from ks1.tb1;",
"field2": "field 2 value"
},
"tags": {
"block": "block0",
"name": "block0__op1",
"name": "op1",
"op": "op1"
}
}
@@ -191,7 +191,7 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"field1": "select * from ks1.tb1;",
"field2": "field 2 value"
@@ -201,7 +201,7 @@ ops:
},
"tags": {
"block": "block0",
"name": "block0__op1",
"name": "op1",
"op": "op1"
}
}
@@ -244,7 +244,7 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"field1": "select * from ks1.tb1;",
"field2": "field 2 value"
@@ -254,7 +254,7 @@ ops:
},
"tags": {
"block": "block0",
"name": "block0__op1",
"name": "op1",
"op": "op1"
}
}
@@ -302,7 +302,7 @@ ops:
[
{
"name": "block0__op1",
"name": "op1",
"op": {
"field1": "select * from ks1.tb1;",
"field2": "field 2 value"
@@ -312,7 +312,7 @@ ops:
},
"tags": {
"block": "block0",
"name": "block0__op1",
"name": "op1",
"op": "op1"
}
}
@@ -351,14 +351,14 @@ ops: "my test op"
"pname": "pvalue"
},
"tags": {
"name": "block0__stmt1",
"name": "stmt1",
"block": "block0",
"op": "stmt1"
},
"op": {
"stmt": "my test op"
},
"name": "block0__stmt1"
"name": "stmt1"
}
]
```
@@ -401,14 +401,14 @@ blocks:
"pname": "pvalue"
},
"tags": {
"name": "block1__stmt1",
"name": "stmt1",
"block": "block1",
"op": "stmt1"
},
"op": {
"stmt": "my test op"
},
"name": "block1__stmt1"
"name": "stmt1"
}
]
```
@@ -458,14 +458,14 @@ blocks:
"pname": "pvalue"
},
"tags": {
"name": "block1__op1",
"name": "op1",
"block": "block1",
"op": "op1"
},
"op": {
"stmt": "my test op"
},
"name": "block1__op1"
"name": "op1"
}
]
```
@@ -520,7 +520,7 @@ blocks:
[
{
"tags": {
"name": "block1__op1",
"name": "op1",
"block": "block1",
"op": "op1"
},
@@ -530,7 +530,7 @@ blocks:
"pname": "pvalue"
}
},
"name": "block1__op1"
"name": "op1"
}
]
```
@@ -583,18 +583,18 @@ blocks:
[
{
"tags": {
"name": "block1__op1",
"name": "op1",
"block": "block1",
"op": "op1"
},
"op": {
"stmt": "my test op"
},
"name": "block1__op1"
"name": "op1"
},
{
"tags": {
"name": "block1__params",
"name": "params",
"block": "block1",
"op": "params"
},
@@ -602,7 +602,7 @@ blocks:
"pname": "pvalue"
},
"name": "block1__params"
"name": "params"
}
]
```

18
adapters-api/src/test/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDetailOverrideTest.java
View File

@@ -43,32 +43,32 @@ public class OpDetailOverrideTest {
OpsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1block0.getOps().size()).isEqualTo(1);
OpTemplate s = doc1block0.getOps().get(0);
assertThat(s.getName()).isEqualTo("block0__stmt1");
assertThat(s.getName()).isEqualTo("stmt1");
assertThat(s.getStmt()).contains("globalstatement1");
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0__stmt1", "op","stmt1"));
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","stmt1", "op","stmt1"));
OpsBlock doc1block1 = doc1.getBlocks().get(1);
List<OpTemplate> ops = doc1block1.getOps();
assertThat(ops).hasSize(4);
s = ops.get(0);
assertThat(s.getName()).isEqualTo("testblock1__stmt1");
assertThat(s.getName()).isEqualTo("stmt1");
assertThat(s.getStmt()).contains("astatement1");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1__stmt1","op","stmt1"));
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","stmt1","op","stmt1"));
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
s = ops.get(1);
assertThat(s.getName()).isEqualTo("testblock1__s2name");
assertThat(s.getName()).isEqualTo("s2name");
assertThat(s.getStmt()).contains("s2statement data");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1__s2name","op","s2name"));
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","s2name","op","s2name"));
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
s = ops.get(2);
assertThat(s.getName()).isEqualTo("testblock1__s3");
assertThat(s.getName()).isEqualTo("s3");
assertThat(s.getStmt()).contains("statement three");
assertThat(s.getTags()).containsEntry("tname1", "tval1");
assertThat(s.getTags()).containsEntry("global_tag1", "tag value");
@@ -76,9 +76,9 @@ public class OpDetailOverrideTest {
assertThat(s.getParams()).hasSize(2);
s = ops.get(3);
assertThat(s.getName()).isEqualTo("testblock1__s4");
assertThat(s.getName()).isEqualTo("s4");
assertThat(s.getStmt()).contains("statement 4");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1__s4","op","s4"));
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","s4","op","s4"));
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);

6
docsys/pom.xml
View File

@@ -50,17 +50,17 @@
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlets</artifactId>
<version>11.0.18</version>
<version>11.0.19</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>11.0.18</version>
<version>11.0.19</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-rewrite</artifactId>
<version>11.0.18</version>
<version>11.0.19</version>
</dependency>
<dependency>

20
engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java
View File

@@ -104,6 +104,7 @@ public class NBAtFile {
filepathSpec=(filepathSpec.endsWith(".yaml") ? filepathSpec : filepathSpec+".yaml");
Path atPath = Path.of(filepathSpec);
String argsdata = "";
try {
argsdata = Files.readString(atPath);
@@ -128,13 +129,28 @@ public class NBAtFile {
throw new RuntimeException("You can not traverse a non-map object type with spec '" + spec + "'");
}
}
return formatted(scopeOfInclude, fmt);
LinkedList<String> formatted = formatted(scopeOfInclude, fmt);
formatted=interposePath(formatted, atPath);
return formatted;
} else {
throw new RuntimeException("Unable to match at-file specifier: " + spec + " to known syntax");
throw new RuntimeException("Unable to match at-file specifier: " + spec + " to pattern '" + includePattern.pattern() + "'");
}
}
private static LinkedList<String> interposePath(LinkedList<String> formatted, Path atPath) {
Path parent = (atPath.getNameCount()==1 ? Path.of(".") : atPath.getParent());
ListIterator<String> iter = formatted.listIterator();
while (iter.hasNext()) {
String word = iter.next();
String modified = word.replaceAll("\\$\\{DIR}",parent.toString());
iter.remove();
iter.add(modified);
}
return formatted;
}
private static LinkedList<String> formatted(Object scopeOfInclude, NBAtFileFormats fmt) {
LinkedList<String> emitted = new LinkedList<>();
if (scopeOfInclude instanceof Map<?,?> map) {

71
engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFileFormats.java
View File

@@ -23,10 +23,46 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public enum NBAtFileFormats {
Default("", s -> s.length <=2, s -> (s.length==1) ? s[0] : s[0] + ":" + s[1]),
MapWithEquals("=", s -> s.length == 2, s -> s[0] + "=" + s[1]),
MapWithColons(":", s -> s.length == 2, s -> s[0] + ":" + s[1]),
GlobalWithDoubleDashes("--", s -> s.length<=2 && s[0].startsWith("--"), NBAtFileFormats::formatDashDashOption);
Default("", s -> s.length <= 2, NBAtFileFormats::formatDefaultDashOption),
MapWithEquals("=", s -> s.length <= 2, NBAtFileFormats::formatNameEqualsValue),
MapWithColons(":", s -> s.length <= 2, NBAtFileFormats::formatNameColonValue),
GlobalWithDoubleDashes("--", s -> s.length <= 2 && s[0].startsWith("--"), NBAtFileFormats::formatDashDashOption);
private static String formatNameEqualsValue(String[] strings) {
if (strings.length == 2) {
return strings[0] + "=" + strings[1];
} else if (strings.length == 1 && strings[0].matches("[a-zA-Z_][a-zA-Z_]*[=:].*")) {
String[] parts = strings[0].split("[=:]", 2);
return parts[0]+"="+parts[1];
} else {
throw new RuntimeException("Unable to match data for namedd value form: " + String.join("|",Arrays.asList(strings)));
}
}
private static String formatNameColonValue(String[] strings) {
if (strings.length == 2) {
return strings[0] + ":" + strings[1];
} else if (strings.length == 1 && strings[0].matches("[a-zA-Z_][a-zA-Z_]*[=:].*")) {
String[] parts = strings[0].split("[=:]", 2);
return parts[0]+":"+parts[1];
} else {
throw new RuntimeException("Unable to match data for namedd value form: " + String.join("|",Arrays.asList(strings)));
}
}
private static String formatDefaultDashOption(String[] strings) {
if (strings.length == 1 && strings[0].startsWith("--")) {
return formatDashDashOption(strings);
} else if (strings.length == 1 && strings[0].matches("[a-zA-Z_][a-zA-Z0-9._]*[=:].*")) {
return formatNameEqualsValue(strings);
} else if (strings.length == 2) {
return formatNameEqualsValue(strings);
} else if (strings.length==1) {
return strings[0];
} else {
throw new RuntimeException("Unable to match data for --global option form: " + String.join("|",Arrays.asList(strings)));
}
}
private final String spec;
private final Predicate<String[]> validator;
@@ -49,33 +85,32 @@ public enum NBAtFileFormats {
public void validate(String[] ary) {
if (!validator.test(ary)) {
throw new RuntimeException("With fmt '" + this.name() + "': input data not valid for format specifier '" + spec + "': data:[" + String.join("],[",Arrays.asList(ary)) + "]");
throw new RuntimeException("With fmt '" + this.name() + "': input data not valid for format specifier '" + spec + "': data:[" + String.join("],[", Arrays.asList(ary)) + "]");
}
}
private final static Pattern doubleOptionSpace = Pattern.compile(
"^(?<optname>--[a-zA-Z][a-zA-Z0-9_.-]*) +(?<optvalue>.+)$"
private final static Pattern doubleDashOption = Pattern.compile(
"^(?<optname>--[a-zA-Z][a-zA-Z0-9_.-]*)((=|\\s+)(?<optvalue>.+))?$"
);
private static String formatDashDashOption(String[] words) {
if (words[0].contains("=")) {
if (words.length==1) {
return words[0];
} else {
throw new RuntimeException("Unrecognized option form " + String.join(" (space) " + Arrays.asList(words)));
}
}
if (words.length>1) {
if (words.length > 1) {
throw new RuntimeException("too many values for rendering --option in at-file: " + Arrays.asList(words));
}
Matcher matcher = doubleOptionSpace.matcher(words[0]);
Matcher matcher = doubleDashOption.matcher(words[0]);
if (matcher.matches()) {
String optname = matcher.group("optname");
String optvalue = matcher.group("optvalue");
return optname + "=" + optvalue;
if (optvalue!=null) {
optvalue = (optvalue.matches("'.+'") ? optvalue.substring(1, optvalue.length() - 1) : optvalue);
optvalue = (optvalue.matches("\".+\"") ? optvalue.substring(1, optvalue.length() - 1) : optvalue);
return optname + "=" + optvalue;
} else {
return optname;
}
} else {
throw new RuntimeException("Unable to convert atfile option: "+ Arrays.asList(words));
throw new RuntimeException("Unable to match option '" + words[0] + "' with pattern " + doubleDashOption.pattern());
}
}

14
engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java
View File

@@ -31,10 +31,16 @@ class NBAtFileTest {
assertThat(strings).containsExactly("arg1","arg2","arg3");
}
@Test
public void testRelativizedPaths() {
LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/relativized.yaml");
assertThat(strings).containsExactly("--option1=src/test/resources/atfiles/value1");
}
@Test
public void testParseSimpleMapDefaultFmt() {
LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/simple_map.yaml");
assertThat(strings).containsExactly("arg1:val1","arg2:val2","arg3:val3");
assertThat(strings).containsExactly("arg1=val1","arg2=val2","arg3=val3");
}
@Test
@@ -67,4 +73,10 @@ class NBAtFileTest {
assertThat(strings).containsExactly("key1=value1","key2=value2");
}
@Test
public void testGlobalOptionForms() {
LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/global_opts.yaml>--");
assertThat(strings).containsExactly("--option1", "--option2=value2", "--option3=value3", "--option4=value4");
}
}

4
engine-cli/src/test/resources/atfiles/global_opts.yaml Normal file
View File

@@ -0,0 +1,4 @@
- --option1
- --option2=value2
- --option3='value3'
- --option4="value4"

1
engine-cli/src/test/resources/atfiles/relativized.yaml Normal file
View File

@@ -0,0 +1 @@
- --option1=${DIR}/value1

7
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/MemInfoReader.java
View File

@@ -46,9 +46,10 @@ public class MemInfoReader extends LinuxSystemFileReader {
public Double getMemUsedkB() {
Double memTotal = getMemTotalkB();
Double memFree = getMemFreekB();
if (memTotal != null && memFree != null)
return memTotal - memFree;
Double memAvailable = getMemAvailablekB();
// Double memFree = getMemFreekB();
if (memTotal != null && memAvailable != null)
return memTotal - memAvailable;
return null;
}

6
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java
View File

@@ -84,6 +84,7 @@ public class NBSession extends NBHeartbeatComponent implements Function<List<Cmd
// TODO: inject container closing commands after the last command referencing each container
List<NBCommandAssembly.CommandInvocation> invocationCalls = NBCommandAssembly.assemble(cmds, this::getContext);
ResultCollector collector = new ResultCollector();
// TODO: When a command is not successful, automatically break out of the command loop
try (ResultContext results = new ResultContext(collector).ok()) {
for (NBCommandAssembly.CommandInvocation invocation : invocationCalls) {
String targetContext = invocation.containerName();
@@ -93,11 +94,14 @@ public class NBSession extends NBHeartbeatComponent implements Function<List<Cmd
NBBufferedContainer container = getContext(targetContext);
NBCommandResult cmdResult = container.apply(command, invocation.params());
results.apply(cmdResult);
if (cmdResult.hasException()) {
throw cmdResult.getException();
}
} catch (Exception e) {
String msg = "While running " + explanation + "', an error occurred: " + e.toString();
results.error(e);
onError(e);
logger.error(msg);
results.error(e);
break;
}
}

13
engine-core/src/main/java/io/nosqlbench/engine/core/logging/NBLoggerConfig.java
View File

@@ -186,6 +186,13 @@ public class NBLoggerConfig extends ConfigurationFactory {
.add(builder.newAppenderRef("console"))
.addAttribute("additivity", false));
if (sessionName == null) {
sessionName = "LOGINIT";
System.out.println("Session was not set in logger config. If you are getting this error, then it means" +
" that the session has not initialized properly, and that some unexpected error occurred before the logging" +
" system was initialized. Look for errors prior to this.");
}
if (sessionName != null) {
if (!Files.exists(loggerDir)) {
@@ -275,7 +282,7 @@ public class NBLoggerConfig extends ConfigurationFactory {
Path logdir = logpath.getParent();
if (Files.exists(logpath)) {
String basename = logpath.getFileName().toString();
String linkname = basename.replace(getSessionName()+"_","");
String linkname = basename.replace(getSessionName() + "_", "");
Path linkPath = logdir.resolve(linkname);
try {
Files.deleteIfExists(linkPath);
@@ -444,7 +451,7 @@ public class NBLoggerConfig extends ConfigurationFactory {
private void attachAuxLogger(ConfigurationBuilder<BuiltConfiguration> builder, String loggerName, Level fileLevel) {
String appenderName = loggerName + (("_APPENDER").toUpperCase());
Path logPath = loggerDir.resolve(getFileBase() + "_" + loggerName.toLowerCase() + ".log");
Path linkPath = loggerDir.resolve(loggerName.toLowerCase()+".log");
Path linkPath = loggerDir.resolve(loggerName.toLowerCase() + ".log");
var appender = builder
.newAppender(appenderName, FileAppender.PLUGIN_NAME)
@@ -463,7 +470,7 @@ public class NBLoggerConfig extends ConfigurationFactory {
try {
Files.deleteIfExists(linkPath);
Files.createSymbolicLink(linkPath,logPath.getFileName());
Files.createSymbolicLink(linkPath, logPath.getFileName());
} catch (IOException e) {
throw new RuntimeException(e);
}

6
mvn-defaults/pom.xml
View File

@@ -80,7 +80,7 @@
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy</artifactId>
<version>4.0.16</version>
<version>4.0.17</version>
</dependency>
<dependency>
<groupId>org.snakeyaml</groupId>
@@ -188,7 +188,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.102.Final</version>
<version>4.1.104.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
@@ -290,7 +290,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.621</version>
<version>1.12.622</version>
</dependency>
<dependency>
<groupId>com.elega9t</groupId>

8
nb-api/src/main/java/io/nosqlbench/nb/api/components/status/NBStatusComponent.java
View File

@@ -40,19 +40,13 @@ public class NBStatusComponent extends NBBaseComponent {
}
public Status status() {
List<Status> subbeats = new ArrayList<>();
StatusVisitor statusVisitor = new StatusVisitor(subbeats);
for (NBComponent child : getChildren()) {
NBComponentTraversal.visitDepthFirstLimited(child,statusVisitor,c -> c instanceof NBStatusComponent);
}
// for (NBComponent child : getChildren()) {
// if (child instanceof NBStatusComponent beat) {
// subbeats.add(beat.heartbeat());
// }
// }
return new Status(
getLabels(),
this.getComponentState(),

3
nbr/src/main/java/io/nosqlbench/scenarios/simframe/stabilization/StabilityDetector.java
View File

@@ -118,6 +118,9 @@ public class StabilityDetector implements Runnable {
// if previous bigger window had a higher stddev than the one after, then it is converging
double reductionFactor = (stddev[i + 1] / stddev[i]);
basis *= reductionFactor;
if (Double.isNaN(basis)) {
throw new RuntimeException("basis is NaN");
}
}
// TODO: investigate why we get NaN sometimes and what it means for stability checks

5
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/vectors/dnn/DNN_euclidean_v_wrap.java
View File

@@ -22,6 +22,11 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
/**
* This represents an enumerated population of vectors of some dimension,
* where any ordinal values which address outside of that enumeration
* simply wrap with modulo.
*/
@ThreadSafeMapper
@Categories(Category.experimental)
public class DNN_euclidean_v_wrap implements LongFunction<float[]> {