IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #942 from nosqlbench/nosqlbench-937

Browse Source 
cleanup duplicitous activity parameters during initialization (yaml,type)
This commit is contained in:
Jonathan Shook 2023-01-23 14:19:18 -06:00 committed by GitHub
commit 362e3c6cd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
55 changed files with 281 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/release.yml → .github/workflows/prerelease.yml vendored
View File

@ -3,7 +3,7 @@ name: release
on:
push:
branches:
- main
- prerelease-testing
paths:
- RELEASENOTES.**

4
adapter-cqld4/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -38,7 +38,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

6
adapter-diag/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -37,13 +37,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<scope>compile</scope>
</dependency>

4
adapter-dynamodb/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -37,7 +37,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

4
adapter-http/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -85,7 +85,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<scope>compile</scope>
</dependency>

6
adapter-kafka/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -87,13 +87,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

4
adapter-mongodb/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -36,7 +36,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

6
adapter-pulsar/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -41,13 +41,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

6
adapter-s4j/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -45,13 +45,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.datastax.oss/pulsar-jms -->

6
adapter-stdout/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -38,14 +38,14 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<scope>compile</scope>
</dependency>

34
adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -34,15 +34,16 @@ import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Service(value= DriverAdapter.class,selector = "stdout")
@Service(value = DriverAdapter.class, selector = "stdout")
public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace> implements SyntheticOpTemplateProvider {
private final static Logger logger = LogManager.getLogger(StdoutDriverAdapter.class);
@Override
public OpMapper<StdoutOp> getOpMapper() {
DriverSpaceCache<? extends StdoutSpace> ctxCache = getSpaceCache();
return new StdoutOpMapper(this,ctxCache);
return new StdoutOpMapper(this, ctxCache);
}
@Override
@ -58,15 +59,18 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
Set<String> activeBindingNames = new LinkedHashSet<>();
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
Set<String> activeBindingNames = new LinkedHashSet<>(stmtsDocList.getDocBindings().keySet());
if (activeBindingNames.size()==0) {
logger.warn("Unable to synthesize op for driver=" + this.getAdapterName() + " with zero bindings.");
return List.of();
}
String bindings = Optional.ofNullable(cfg.get("bindings")).map(Object::toString).orElse("doc");
activeBindingNames.addAll(stmtsDocList.getDocBindings().keySet());
Pattern bindingsFilter = Pattern.compile(bindings.equalsIgnoreCase("doc") ? ".*" : bindings);
Set<String> filteredBindingNames = new LinkedHashSet<>();
activeBindingNames
Set<String> filteredBindingNames = activeBindingNames
.stream()
.filter(n -> {
if (bindingsFilter.matcher(n).matches()) {
@ -77,16 +81,20 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
return false;
}
})
.forEach(filteredBindingNames::add);
activeBindingNames = filteredBindingNames;
.collect(Collectors.toSet());
if (filteredBindingNames.size() == 0) {
logger.warn("Unable to synthesize op for driver="+getAdapterName()+" when " + activeBindingNames.size()+"/"+activeBindingNames.size() + " bindings were filtered out with bindings=" + bindings);
return List.of();
}
OpData op = new OpData("synthetic", "synthetic", Map.of(), stmtsDocList.getDocBindings(), cfg,
Map.of("stmt", genStatementTemplate(activeBindingNames, cfg)));
Map.of("stmt", genStatementTemplate(filteredBindingNames, cfg)));
return List.of(op);
}
private String genStatementTemplate(Set<String> keySet, Map<String,Object> cfg) {
private String genStatementTemplate(Set<String> keySet, Map<String, Object> cfg) {
TemplateFormat format = Optional.ofNullable(cfg.get("format"))
.map(Object::toString)
.map(TemplateFormat::valueOf)

6
adapter-tcp/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -40,12 +40,12 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-stdout</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<scope>compile</scope>
</dependency>

8
adapters-api/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -39,19 +39,19 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-spectest</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-userlibs</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>

6
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -31,6 +31,10 @@ public class RawStmtsDocList {
return new RawStmtsDocList(List.of(rawStmtsDoc));
}
public static RawStmtsDocList none() {
return new RawStmtsDocList(List.of());
}
public List<RawStmtsDoc> getStmtsDocs() {
return rawStmtsDocList;
}

7
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -41,6 +41,10 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
this.rawStmtsDocList = rawStmtsDocList;
}
public static StmtsDocList none() {
return new StmtsDocList(RawStmtsDocList.none());
}
public List<StmtsDoc> getStmtDocs(String tagFilter) {
TagFilter tf = new TagFilter(tagFilter);
return getStmtDocs().stream()
@ -67,7 +71,6 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
TagFilter ts = new TagFilter(tagFilterSpec);
List<OpTemplate> opTemplates = new ArrayList<>();
getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream())
.filter(ts::matchesTagged)

4
adapters-api/src/main/java/io/nosqlbench/engine/api/templating/OpTemplateSupplier.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,7 +26,7 @@ import java.util.Optional;
* An Op Template Supplier can provide its own source of op templates instead
* of relying on the built-in mechanism. By default, the built-in mechanism
* will read op definitions from parameters first, then any ops (statements)
* from yaml files provided in the workload= or yaml= activity parameters.
* from yaml files provided in the workload= activity parameters.
*/
public interface OpTemplateSupplier extends DriverAdapterDecorators {

24
adapters-api/src/main/java/io/nosqlbench/engine/api/util/TagFilter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,7 +18,10 @@ package io.nosqlbench.engine.api.util;
import io.nosqlbench.api.engine.util.Tagged;
import java.util.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -98,6 +101,19 @@ public class TagFilter {
private final static Pattern conjugateForm = Pattern.compile("^(?<conjugate>\\w+)\\((?<filter>.+)\\)$",Pattern.DOTALL|Pattern.MULTILINE);
public <T extends Tagged> List<T> filter(List<T> tagged) {
return tagged.stream()
.filter(this::matchesTagged)
.collect(Collectors.toList());
}
public <T extends Tagged> List<String> filterLog(List<T> tagged) {
return tagged.stream()
.map(this::matchesTaggedResult)
.map(Result::getLog)
.collect(Collectors.toList());
}
private enum Conjugate {
any((i,j) -> (j>0)),
all((i,j) -> (i.intValue()==j.intValue())),
@ -201,10 +217,6 @@ public class TagFilter {
return new Result(matched, log);
}
public Result matchesMap(Map<String, String> tags) {
return matches(tags);
}
public Result matchesTaggedResult(Tagged item) {
return matches(item.getTags());
}

6
docsys/pom.xml
View File

@ -28,7 +28,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -37,7 +37,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<!-- jetty -->
@ -128,7 +128,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

12
engine-api/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -39,31 +39,31 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-spectest</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-userlibs</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

34
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -38,6 +38,7 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
@ -193,7 +194,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
}
public String toString() {
return getAlias()+":"+getRunState()+":"+getRunStateTally().toString();
return getAlias() + ":" + getRunState() + ":" + getRunStateTally().toString();
}
@Override
@ -486,7 +487,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
.orElse(SequencerType.bucket);
SequencePlanner<OpDispenser<? extends O>> planner = new SequencePlanner<>(sequencerType);
int dryrunCount=0;
int dryrunCount = 0;
for (int i = 0; i < pops.size(); i++) {
long ratio = ratios.get(i);
ParsedOp pop = pops.get(i);
@ -510,7 +511,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
// }
planner.addOp((OpDispenser<? extends O>) dispenser, ratio);
}
if (dryrunCount>0) {
if (dryrunCount > 0) {
logger.warn("initialized " + dryrunCount + " op templates for dry run only. These ops will be synthesized for each cycle, but will not be executed.");
}
@ -547,22 +548,33 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
protected List<OpTemplate> loadOpTemplates(Optional<DriverAdapter> defaultDriverAdapter) {
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
// StrInterpolator interp = new StrInterpolator(activityDef);
StmtsDocList stmtsDocList = loadStmtsDocList();
List<OpTemplate> unfilteredOps = stmtsDocList.getStmts();
List<OpTemplate> filteredOps = stmtsDocList.getStmts(tagfilter);
if (filteredOps.size() == 0) {
if (unfilteredOps.size() > 0) {
if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out
throw new BasicError("There were no active statements with tag filter '"
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
} else {
// There were no ops, and it *wasn't* because they were all filtered out.
// In this case, let's try to synthesize the ops as long as at least a default driver was provided
if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
filteredOps = sotp.getSyntheticOpTemplates(stmtsDocList, getActivityDef().getParams());
Objects.requireNonNull(filteredOps);
if (filteredOps.size() == 0) {
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + "'" +
" but no ops were created. You must provide either a workload or an op parameter. Activities require op templates.");
}
} else { // But if there were no ops, and there was no default driver provided, we can't continue
throw new BasicError("""
No op templates were provided. You must provide one of these activity parameters:
1) workload=some.yaml
2) op='inline template
3) driver=stdout (or any other drive that can synthesize ops)""");
}
}
if (filteredOps.size() == 0) {
@ -652,19 +664,17 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
protected StmtsDocList loadStmtsDocList() {
try {
StmtsDocList stmtsDocList = null;
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
stmtsDocList = StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
workloadSource = "commandline:" + stmt.get();
return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
} else if (op_yaml_loc.isPresent()) {
stmtsDocList = StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
workloadSource = "yaml:" + op_yaml_loc.get();
return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
}
return stmtsDocList;
return StmtsDocList.none();
} catch (Exception e) {
throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);

28
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateTally.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,8 +20,6 @@ import io.nosqlbench.engine.api.activityapi.core.RunState;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
/**
* <H2>Synopsis</H2>
* <P>Event-oriented tally of the runtime states of all
@ -90,9 +88,9 @@ public class RunStateTally {
logger.trace(() -> this +" -"+from+ String.format(":%04d",counts[from.ordinal()])+ ", +"+to+ String.format(":%04d",counts[to.ordinal()]));
if (counts[from.ordinal()]==0 || counts[to.ordinal()]==1) {
logger.debug(() -> "NOTIFYing on edge "+
"from " + from + String.format(":%04d",counts[from.ordinal()]) +
" to " + to + String.format(":%04d",counts[to.ordinal()]));
// logger.debug(() -> "NOTIFYing on edge "+
// "from " + from + String.format(":%04d",counts[from.ordinal()]) +
// " to " + to + String.format(":%04d",counts[to.ordinal()]));
notifyAll();
}
}
@ -107,7 +105,7 @@ public class RunStateTally {
counts[state.ordinal()]++;
logger.trace(() -> this +" +"+state+ String.format(":%04d",counts[state.ordinal()]));
if (counts[state.ordinal()]==1) {
logger.debug(() -> "NOTIFYing on ++-SOME edge for " + state + String.format(":%04d",counts[state.ordinal()]));
// logger.debug(() -> "NOTIFYing on ++-SOME edge for " + state + String.format(":%04d",counts[state.ordinal()]));
notifyAll();
}
}
@ -122,7 +120,7 @@ public class RunStateTally {
counts[state.ordinal()]--;
logger.trace(() -> this +" -"+state+ String.format(":%04d",counts[state.ordinal()]));
if (counts[state.ordinal()]==0) {
logger.debug(() -> "NOTIFYing on 00-NONE edge for " + state + String.format(":%04d",counts[state.ordinal()]));
// logger.debug(() -> "NOTIFYing on 00-NONE edge for " + state + String.format(":%04d",counts[state.ordinal()]));
notifyAll();
}
}
@ -145,7 +143,7 @@ public class RunStateTally {
* @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
*/
public synchronized RunStateImage awaitNoneOther(long timeoutMillis, RunState... runStates) {
logger.debug(() -> "☐ Awaiting only " + Arrays.toString(runStates) + " for " + timeoutMillis+"ms");
// logger.debug(() -> "☐ Awaiting only " + Arrays.toString(runStates) + " for " + timeoutMillis+"ms");
long timeoutAt = timeoutAt(timeoutMillis);
int sum=0;
@ -171,7 +169,7 @@ public class RunStateTally {
}
boolean timedout = (sum!=0);
logger.debug(() -> (timedout ? "✘ TIMED-OUT awaiting only " : "☑ Awaited only " ) + toString(runStates));
// logger.debug(() -> (timedout ? "✘ TIMED-OUT awaiting only " : "☑ Awaited only " ) + toString(runStates));
return new RunStateImage(this.counts,timedout);
}
@ -194,7 +192,7 @@ public class RunStateTally {
* @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
*/
public synchronized RunStateImage awaitNoneOf(long timeoutMillis, RunState... runStates) {
logger.debug(() -> "☐ Awaiting none of " + Arrays.toString(runStates)+ " for " + timeoutMillis+"ms");
// logger.debug(() -> "☐ Awaiting none of " + Arrays.toString(runStates)+ " for " + timeoutMillis+"ms");
long timeoutAt = timeoutAt(timeoutMillis);
int sum=0;
@ -212,7 +210,7 @@ public class RunStateTally {
}
}
boolean timedout=sum==0;
logger.debug(() -> (timedout ? "✘ TIMED-OUT awaiting none of " : "☑ Awaited none of " ) + toString(runStates));
// logger.debug(() -> (timedout ? "✘ TIMED-OUT awaiting none of " : "☑ Awaited none of " ) + toString(runStates));
return new RunStateImage(this.counts,timedout);
}
@ -233,13 +231,13 @@ public class RunStateTally {
* @return A {@link RunStateImage}, indicating success or failure, and the view of states at the time of evaluation
*/
public synchronized RunStateImage awaitAny(long timeoutMillis, RunState... runStates) {
logger.debug(() -> "☐ Awaiting any " + Arrays.toString(runStates) + " for " + timeoutMillis + "ms");
// logger.debug(() -> "☐ Awaiting any " + Arrays.toString(runStates) + " for " + timeoutMillis + "ms");
long timeoutAt = timeoutAt(timeoutMillis);
while (System.currentTimeMillis()<timeoutAt) {
for (RunState runState : runStates) {
if (counts[runState.ordinal()]>0) {
logger.debug("☑ Awaited any " + toString(runStates));
// logger.debug("☑ Awaited any " + toString(runStates));
return new RunStateImage(this.counts,false);
}
}
@ -248,7 +246,7 @@ public class RunStateTally {
} catch (InterruptedException e) {
}
}
logger.debug(() -> "✘ TIMED-OUT awaiting any of " + toString(runStates));
// logger.debug(() -> "✘ TIMED-OUT awaiting any of " + toString(runStates));
return new RunStateImage(this.counts,true);
}

10
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -54,7 +54,6 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
public StandardActivity(ActivityDef activityDef) {
super(activityDef);
this.adapters.putAll(adapters);
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (yaml_loc.isPresent()) {
@ -69,11 +68,14 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
Optional<DriverAdapter> defaultAdapter = activityDef.getParams().getOptionalString("driver")
.flatMap(s -> ServiceSelector.of(s, adapterLoader).get());
// HERE, op templates are loaded before drivers are loaded
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter);
List<ParsedOp> pops = new ArrayList<>();
List<DriverAdapter> adapterlist = new ArrayList<>();
NBConfigModel supersetConfig = ConfigModel.of(StandardActivity.class).add(yamlmodel);
for (OpTemplate ot : opTemplates) {
ParsedOp incompleteOpDef = new ParsedOp(ot, NBConfiguration.empty(), List.of());
String driverName = incompleteOpDef.takeOptionalStaticValue("driver", String.class)
@ -94,6 +96,8 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
if (adapter instanceof NBConfigurable configurable) {
NBConfigModel adapterModel = configurable.getConfigModel();
supersetConfig.add(adapterModel);
combinedModel = adapterModel.add(yamlmodel);
combinedConfig = combinedModel.matchConfig(activityDef.getParams());
configurable.applyConfig(combinedConfig);
@ -101,6 +105,8 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
adapters.put(driverName, adapter);
mappers.put(driverName, adapter.getOpMapper());
}
supersetConfig.assertValidConfig(activityDef.getParams().getStringStringMap());
DriverAdapter adapter = adapters.get(driverName);
adapterlist.add(adapter);

5
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,7 +21,6 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.config.params.Synonyms;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.content.NBPathsAPI;
@ -94,7 +93,6 @@ public class NBCLIScenarioParser {
&& arglist.peekFirst().contains("=")
&& !arglist.peekFirst().startsWith("-")) {
String[] arg = arglist.removeFirst().split("=", 2);
arg[0] = Synonyms.canonicalize(arg[0], logger);
if (userProvidedParams.containsKey(arg[0])) {
throw new BasicError("duplicate occurrence of option on command line: " + arg[0]);
}
@ -236,7 +234,6 @@ public class NBCLIScenarioParser {
throw new BasicError("Unable to recognize scenario cmd spec in '" + commandFragment + "'");
}
String commandName = matcher.group("name");
commandName = Synonyms.canonicalize(commandName, logger);
String assignmentOp = matcher.group("oper");
String assignedValue = matcher.group("val");
parsedStep.put(commandName, new CmdArg(commandName, assignmentOp, assignedValue));

6
engine-cli/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -39,13 +39,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-core</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-docker</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

4
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -71,7 +71,7 @@ public class NBCLIScenarioParserTest {
@Test
public void testThatVerboseFinalParameterThrowsError() {
assertThatExceptionOfType(BasicError.class)
.isThrownBy(() -> new NBCLIOptions(new String[]{ "scenario-test", "yaml=canttouchthis"}));
.isThrownBy(() -> new NBCLIOptions(new String[]{ "scenario-test", "workload=canttouchthis"}));
}
@Test

8
engine-cli/src/test/resources/activities/scenario-test.yaml
View File

@ -2,11 +2,11 @@ min_version: "4.17.15"
scenarios:
default:
schema: run driver==stdout yaml===scenario-test tags=block:schema
rampup: run driver=stdout yaml===scenario-test tags=block:rampup cycles=TEMPLATE(cycles1,10)
main: run driver=stdout yaml===scenario-test tags=block:"main.*" cycles=TEMPLATE(cycles2,10)
schema: run driver==stdout workload===scenario-test tags=block:schema
rampup: run driver=stdout workload===scenario-test tags=block:rampup cycles=TEMPLATE(cycles1,10)
main: run driver=stdout workload===scenario-test tags=block:"main.*" cycles=TEMPLATE(cycles2,10)
schema-only:
- "run driver=stdout yaml=scenario-test tags=phase:schema doundef==undef"
- "run driver=stdout workload=scenario-test tags=phase:schema doundef==undef"
template-test:
with-template: run driver=stdout cycles=TEMPLATE(cycles-test,10)

4
engine-core/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -38,7 +38,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

16
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ExecutionResult.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -38,13 +38,13 @@ public class ExecutionResult {
this.exception = error;
this.iolog = ((iolog != null) ? iolog + "\n\n" : "") + exception;
logger.debug("populating "+(error==null ? "NORMAL" : "ERROR")+" scenario result");
if (logger.isTraceEnabled()) {
StackTraceElement[] st = Thread.currentThread().getStackTrace();
for (int i = 0; i < st.length; i++) {
logger.debug(":AT " + st[i].getFileName()+":"+st[i].getLineNumber()+":"+st[i].getMethodName());
if (i>10) break;
}
}
// if (logger.isTraceEnabled()) {
// StackTraceElement[] st = Thread.currentThread().getStackTrace();
// for (int i = 0; i < st.length; i++) {
// logger.debug(":AT " + st[i].getFileName()+":"+st[i].getLineNumber()+":"+st[i].getMethodName());
// if (i>10) break;
// }
// }
}

4
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivitiesExceptionHandler.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -28,7 +28,7 @@ public class ActivitiesExceptionHandler implements Thread.UncaughtExceptionHandl
public ActivitiesExceptionHandler(ScenarioController controller) {
this.controller = controller;
logger.debug(() -> "Activity exception handler starting up for executor '" + this.controller + "'");
logger.debug(() -> "Activities exception handler starting up for executor '" + this.controller + "'");
}

4
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExceptionHandler.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -27,7 +27,7 @@ public class ActivityExceptionHandler implements Thread.UncaughtExceptionHandler
public ActivityExceptionHandler(ActivityExecutor executor) {
this.executor = executor;
logger.debug(() -> "Activity exception handler starting up for executor '" + executor + "'");
logger.debug(() -> "Activity executor exception handler starting up for executor '" + executor.getActivityDef().getAlias() + "'");
}

4
engine-docker/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -72,7 +72,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

4
engine-docs/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -44,7 +44,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>docsys</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

4
engine-extensions/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -38,7 +38,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

4
mvn-defaults/pom.xml
View File

@ -19,12 +19,12 @@
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<packaging>pom</packaging>
<properties>
<revision>5.17.0-SNAPSHOT</revision>
<revision>5.17.1-SNAPSHOT</revision>
<!-- Set this level to override the logging level for tests during build -->
<project.testlevel>INFO</project.testlevel>
<!-- Set this level to override the logging level for tests logging configuration during build -->

2
nb-annotations/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>

4
nb-api/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -57,7 +57,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

12
nb-api/src/main/java/io/nosqlbench/api/config/params/ParamsParser.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -255,16 +255,6 @@ public class ParamsParser {
throw new RuntimeException("Unable to parse input:" + input);
}
if (canonicalize) {
List<String> keys = new ArrayList<>(parms.keySet());
for (String key : keys) {
String properkey = Synonyms.canonicalize(key, logger);
if (!key.equals(properkey)) {
parms.put(properkey, parms.get(key));
parms.remove(key);
}
}
}
return parms;
}

72
nb-api/src/main/java/io/nosqlbench/api/config/params/Synonyms.java
View File

@ -1,72 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.config.params;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.regex.Pattern;
/**
* This class is just a central reference point for the names of parameters
* or other configuration-level primitives which have been given better names.
* For the sake of backwards compatibility, the old names are retained, but
* deprecated and warned against.
*/
public class Synonyms {
/**
* Each entry in this list is a list of synonyms in configuration.
*/
public final static Map<String, Set<String>> PARAM_SYNONYMS = new HashMap<>() {{
put("hosts", Set.of("host" ));
put("workload", Set.of("yaml" ));
put("driver", Set.of("type" ));
put("rate", Set.of("targetrate", "cyclerate" ));
put("parameterized", Set.of("parametrized" )); // mispelling safety net
}};
/**
* use this method to convert deprecated
* @param input A configuration string from a user or file
* @param synonyms A list of known synonym lists with the preferred values first, like {@link #PARAM_SYNONYMS}
* @param warnings An BiConsumer which can handle (deprecated, preferred) for subsitutions.
* @return The configuration string in canonicalized form, with the preferred names used where possible
*/
public static String canonicalize(String input, Map<String, Set<String>> synonyms, BiConsumer<String,String> warnings) {
String replaced = input;
for (Map.Entry<String, Set<String>> syns : synonyms.entrySet()) {
String preferred = syns.getKey();
for (String deprecated : syns.getValue()) {
Pattern p = Pattern.compile("\\b" + deprecated + "\\b");
String prior = replaced;
replaced = replaced.replaceAll(p.pattern(),preferred);
if (!prior.equals(replaced) && warnings!=null) {
warnings.accept(deprecated,preferred);
}
}
}
return replaced;
}
public static String canonicalize(String arg, Logger logger) {
return canonicalize(arg, PARAM_SYNONYMS, (d, p) -> logger.debug(() ->
"rewrote synonym to canonical term (" + d +" => " + p +")"
));
}
}

11
nb-api/src/main/java/io/nosqlbench/api/config/standard/ConfigModel.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,6 @@
package io.nosqlbench.api.config.standard;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.api.errors.BasicError;
import java.math.BigDecimal;
@ -47,14 +46,6 @@ public class ConfigModel implements NBConfigModel {
return new ConfigModel(ofType);
}
public static NBConfiguration defacto(ActivityDef def) {
ConfigModel configModel = new ConfigModel(Object.class);
for (Map.Entry<String, Object> entry : def.getParams().entrySet()) {
configModel.add(Param.defaultTo(entry.getKey(), entry.getValue().toString()));
}
return configModel.apply(def.getParams());
}
public <T> ConfigModel add(Param<T> param) {
this.params.add(param);
for (String name : param.getNames()) {

38
nb-api/src/test/java/io/nosqlbench/nb/api/config/SynonymsTest.java
View File

@ -1,38 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.config;
import io.nosqlbench.api.config.params.Synonyms;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class SynonymsTest {
@Test
public void testCanonicalizeText() {
String before = "test yaml ayamlfoo type btype typea targetrate";
StringBuilder sb = new StringBuilder();
String result = Synonyms.canonicalize(before, Synonyms.PARAM_SYNONYMS,
(s, s2) -> sb.append("replaced ").append(s).append(" with ").append(s2).append("\n"));
assertThat(result).isEqualTo("test workload ayamlfoo driver btype typea rate");
assertThat(sb.toString()).isEqualTo("replaced type with driver\n" +
"replaced targetrate with rate\n" +
"replaced yaml with workload\n");
}
}

2
nb-spectest/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>

26
nb5/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -40,7 +40,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nbr</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
@ -56,37 +56,37 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-tcp</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-mongodb</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-stdout</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-diag</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-dynamodb</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-cqld4</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
@ -98,13 +98,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-http</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pulsar</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
@ -116,13 +116,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-s4j</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-kafka</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
@ -245,7 +245,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-mongodb</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>

6
nbr-examples/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -39,13 +39,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nbr</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-diag</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

2
nbr-examples/src/test/java/io/nosqlbench/nbr/examples/ScriptExampleTests.java
View File

@ -275,7 +275,7 @@ public class ScriptExampleTests {
public void testErrorPropagationFromActivityInitialization() {
ExecutionMetricsResult scenarioResult = runScenario("activity_init_error");
assertThat(scenarioResult.getException()).isNotNull();
assertThat(scenarioResult.getException().getMessage()).contains("Unable to convert end cycle from invalid");
assertThat(scenarioResult.getException().getMessage()).contains("Unknown config parameter 'unknown_config'");
assertThat(scenarioResult.getException()).isNotNull();
}

12
nbr/pom.xml
View File

@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -40,31 +40,31 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-cli</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-docs</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-core</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-extensions</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-diag</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

2
pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>mvn-defaults</relativePath>
</parent>

61
scripts/release-revision.sh Executable file
View File

@ -0,0 +1,61 @@
#!/bin/bash
#
# Copyright (c) 2023 nosqlbench
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
set -x
export REVISION=$(mvn help:evaluate -Dexpression=revision -q -DforceStdout)
export PRERELEASE_REVISION=$(echo "${REVISION}" | cut -d'-' -f1)
printf "REVISION: '%s' PRERELEASE_REVISION: '%s'\n" "${REVISION}" "${PRERELEASE_REVISION}"
mvn package -Drevision="${PRERELEASE_REVISION}"
exit
GIT_RELEASE_BOT_NAME=${GIT_RELEASE_BOT_NAME:?GIT_RELEASE_BOT_NAME must be provided}
GITHUB_SHA=${GITHUB_SHA:?GITHUB_SHA must be provided}
GITHUB_REF=${GITHUB_REF:?GITHUB_REF must be provided}
RELEASE_BRANCH_PATTERN=${RELEASE_BRANCH_PATTERN:?RELEASE_BRANCH_PATTERN must be provided}
PRERELEASE_BRANCH_PATTERN=${PRERELEASE_BRANCH_PATTERN:?PRERELEASE_BRANCH_PATTERN must be provided}
#git rev-parse --abbrev-ref HEAD
# Filter the branch to execute the release on
readonly local current_branch=$(git rev-parse --abbrev-ref HEAD)
echo "Current branch: ${current_branch}"
if [[ -n "${current_branch}" && "${current_branch}" == *"${RELEASE_BRANCH_PATTERN}"* ]]; then
echo "Building for release branch ${current_branch}"
elif [[ -n "${current_branch}" && "${current_branch}" == *"${PRERELEASE_BRANCH_PATTERN}"* ]]; then
echo "Building prerelease for branch ${current_branch}"
else
echo "Skipping for ${current_branch} branch"
exit 0
fi
# Making sure we are on top of the branch
echo "Git checkout branch ${GITHUB_REF##*/}"
git checkout ${GITHUB_REF##*/}
echo "Git reset hard to ${GITHUB_SHA}"
git reset --hard ${GITHUB_SHA}
# Do the release
echo "Do mvn release:prepare..."
#mvn $MAVEN_REPO_LOCAL --batch-mode --global-settings release.xml -Dusername=$GITHUB_ACCESS_TOKEN release:prepare
mvn --batch-mode --global-settings release.xml -Dusername=$GITHUB_ACCESS_TOKEN clean release:prepare -DdevelopmentVersion=${NEXT_SNAPSHOT} -DreleaseVersion=${RELEASE_VERSION}
echo "files after release:prepare..."
pwd
ls -l

6
virtdata-api/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -39,14 +39,14 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<artifactId>nb-api</artifactId>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lang</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>

2
virtdata-lang/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>

4
virtdata-lib-basics/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -36,7 +36,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

6
virtdata-lib-curves4/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -38,13 +38,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-basics</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

6
virtdata-lib-random/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -36,13 +36,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-basics</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

4
virtdata-lib-realer/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -36,7 +36,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-basics</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
</dependencies>

4
virtdata-realdata/pom.xml
View File

@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -34,7 +34,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>

16
virtdata-userlibs/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -34,42 +34,42 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-realdata</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-realer</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-random</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-basics</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lib-curves4</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>docsys</artifactId>
<version>5.17.1-SNAPSHOT</version>
<version>${revision}</version>
<scope>compile</scope>
</dependency>