IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

api for annotations

Browse Source
This commit is contained in:
Jonathan Shook 2020-11-16 17:35:04 -06:00
parent c2cec2357c
commit 317ffab49c
13 changed files with 624 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
devdocs/sketches/annotations.md
View File

@ -4,7 +4,7 @@ NOTE: Here, annotations are notes that are stored in a metrics system for
review, not _Java Annotations_.
The annotations support in nosqlbench is meant to allow for automatic
annotation of important timestamps and qualifying details for a
grafanaAnnotation of important timestamps and qualifying details for a
nosqlbench scenario.
# Annotation Semantics
@ -18,38 +18,72 @@ Annotations always have at least one timestamp, and up to two
. Annotations with one timestamp mark an instant where an event
is known to have occurred.
When instrumenting an event for annotation, both positive and negative
When instrumenting an event for grafanaAnnotation, both positive and negative
outcomes must be instrumented. That is, if a user is expecting an
annotation marker for when an activity was started, they should
instead see an error annotation if there indeed was an error. The
grafanaAnnotation marker for when an activity was started, they should
instead see an error grafanaAnnotation if there indeed was an error. The
successful outcome of starting an activity is a different event
than the failure of it, but they both speak to the outcome of
trying to start an activity.
# NoSQLBench Event Taxonomy
# NoSQLBench Annotation Level
Each annotation comes from a particular level of execution with
NoSQLBench. Starting from the top, each layer is nested within
the last. The conceptual view of this would appear as:
+--------+
| op |
+------------+
| motor |
+-----------------+
| activity |
+---------------------+
| scripting |
+-------------------------+ +---------------+
| scenario | | application |
+-------------------------------------------------+
| CLI ( Command Line Interface ) |
+-------------------------------------------------+
That is, every op happens within a thread motor, every thread motor
happens within an activity, and so on.
- cli
- cli.render
- cli.execution
- cli.error
- cli.render - When the CLI renders a scenario script
- cli.execution - When the CLI executes a scenario
- cli.error - When there is an error at the CLI level
- scenario
- scenario.start
- scenario.stop
- scenario.error
- scenario.params - When a scenario is configured with parameters
- scenario.start - When a scenario is started
- scenario.stop - When a scenario is stopped
- scenario.error - When a scenario throws an error
- scripting
- extensions - When an extension service object is created
- activity
- activity.start
- activity.stop
- activity.param
- activity.error
- thread
- thread.state
- thread.error
- user
- note
- extension
- activity.params - When params are initially set or changed
- activity.start - Immediately before an activity is started
- activity.stop - When an activity is stopped
- activity.error - When an activity throws an error
- motor
- thread.state - When a motor thread changes state
- thread.error - When a motor thread throws an error
- op
-- There are no op-level events at this time
- application
-- There are no application-level events at this time
## tags
These standard tags should be added to every annotation emitted by
NoSQLBench:
**appname**: "nosqlbench"
**layer**: one of the core layers as above
**event**: The name of the event within the layer as shown above
type
: <specific event name>
layer

20
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -18,7 +18,8 @@ import io.nosqlbench.engine.core.script.Scenario;
import io.nosqlbench.engine.core.script.ScenariosExecutor;
import io.nosqlbench.engine.core.script.ScriptParams;
import io.nosqlbench.engine.docker.DockerMetricsManager;
import io.nosqlbench.nb.api.annotation.Annotator;
import io.nosqlbench.nb.api.annotations.Annotation;
import io.nosqlbench.nb.api.Layer;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
@ -89,6 +90,8 @@ public class NBCLI {
boolean dockerMetrics = globalOptions.wantsDockerMetrics();
String dockerMetricsAt = globalOptions.wantsDockerMetricsAt();
String reportGraphiteTo = globalOptions.wantsReportGraphiteTo();
String annotatorsConfig = globalOptions.getAnnotatorsConfig();
int mOpts = (dockerMetrics ? 1 : 0) + (dockerMetricsAt != null ? 1 : 0) + (reportGraphiteTo != null ? 1 : 0);
if (mOpts > 1 && (reportGraphiteTo == null || annotatorsConfig == null)) {
throw new BasicError("You have multiple conflicting options which attempt to set\n" +
@ -249,6 +252,16 @@ public class NBCLI {
System.exit(0);
}
Annotators.init(annotatorsConfig);
Annotators.recordAnnotation(
Annotation.newBuilder()
.session(sessionName)
.now()
.layer(Layer.CLI)
.detail("cli", Strings.join(args, "\n"))
.build()
);
if (reportGraphiteTo != null || options.wantsReportCsvTo() != null) {
MetricReporters reporters = MetricReporters.getInstance();
reporters.addRegistry("workloads", ActivityMetrics.getMetricRegistry());
@ -262,11 +275,6 @@ public class NBCLI {
reporters.start(10, options.getReportInterval());
}
Annotators.recordAnnotation(sessionName,
Map.of("event", "command-line", "args", String.join(" ", args)),
Map.of()
);
if (options.wantsEnableChart()) {
logger.info("Charting enabled");
if (options.getHistoLoggerConfigs().size() == 0) {

128
engine-core/src/main/java/io/nosqlbench/engine/core/annotation/Annotators.java
View File

@ -1,60 +1,118 @@
package io.nosqlbench.engine.core.annotation;
import io.nosqlbench.nb.api.annotation.Annotator;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.annotations.Annotation;
import io.nosqlbench.nb.api.annotations.Annotator;
import io.nosqlbench.nb.api.config.ConfigAware;
import io.nosqlbench.nb.api.config.ConfigLoader;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.*;
/**
* Singleton-scoped annotator interface for the local process.
* This uses SPI to find the annotators and some config scaffolding
* to make configuring them easier.
* Any number of annotators is allowed of any supporting interface.
* Each instance of a config is used to initialize a single annotator,
* and annotations are distributed to each of them in turn.
*/
public class Annotators {
private final static Logger logger = LogManager.getLogger("ANNOTATORS");
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private static List<Annotator> annotators;
private static Set<String> names;
/**
* Initialize the active annotators.
* Initialize the active annotators. This method must be called before any others.
*
* @param annotatorsConfig A comma-separated set of annotator configs, each with optional
* configuration metadata in name{config} form.
* @param annotatorsConfig A (possibly empty) set of annotator configurations, in any form
* supported by {@link ConfigLoader}
*/
public synchronized static void init(String annotatorsConfig) {
if (annotatorsConfig == null || annotatorsConfig.isEmpty()) {
Annotators.names = Set.of();
} else {
ConfigLoader loader = new ConfigLoader();
annotators = new ArrayList<>();
LinkedHashMap<String, ServiceLoader.Provider<Annotator>> providers = getProviders();
List<Map> configs = loader.load(annotatorsConfig, Map.class);
if (configs != null) {
for (Map cmap : configs) {
Object typeObj = cmap.remove("type");
String typename = typeObj.toString();
ServiceLoader.Provider<Annotator> annotatorProvider = providers.get(typename);
if (annotatorProvider == null) {
throw new RuntimeException("Annotation provider with selector '" + typename + "' was not found.");
}
Annotator annotator = annotatorProvider.get();
if (annotator instanceof ConfigAware) {
ConfigAware configAware = (ConfigAware) annotator;
configAware.applyConfig(cmap);
}
annotators.add(annotator);
}
}
Annotators.names = names;
logger.debug("Initialized " + Annotators.annotators.size() + " annotators, since the configuration is empty.");
}
public synchronized static List<Annotator> getAnnotators() {
if (names == null) {
throw new RuntimeException("Annotators.init(...) must be called first.");
private static List<Annotator> getAnnotators() {
if (annotators != null) {
return annotators;
}
if (annotators == null) {
annotators = new ArrayList<>();
ServiceLoader<Annotator> loader = ServiceLoader.load(Annotator.class);
loader.stream()
.map(sp -> sp.get())
.filter(an -> names.contains("all") || Annotators.names.contains(an.getName()))
.forEach(an -> {
annotators.add(an);
});
}
return annotators;
logger.debug("Annotations are bypassed as no annotators were configured.");
return List.of();
}
public static synchronized void recordAnnotation(
String sessionName,
long startEpochMillis,
long endEpochMillis,
Map<String, String> target,
Map<String, String> details) {
getAnnotators().forEach(a -> a.recordAnnotation(sessionName, startEpochMillis, endEpochMillis, target, details));
private synchronized static LinkedHashMap<String, ServiceLoader.Provider<Annotator>> getProviders() {
ServiceLoader<Annotator> loader = ServiceLoader.load(Annotator.class);
LinkedHashMap<String, ServiceLoader.Provider<Annotator>> providers;
providers = new LinkedHashMap<>();
loader.stream().forEach(provider -> {
Class<? extends Annotator> type = provider.type();
if (!type.isAnnotationPresent(Service.class)) {
throw new RuntimeException(
"Annotator services must be annotated with distinct selectors\n" +
"such as @Service(Annotator.class,selector=\"myimpl42\")"
);
}
Service service = type.getAnnotation(Service.class);
providers.put(service.selector(), provider);
});
return providers;
}
public static synchronized void recordAnnotation(
String sessionName,
Map<String, String> target,
Map<String, String> details) {
recordAnnotation(sessionName, 0L, 0L, target, details);
public static synchronized void recordAnnotation(Annotation annotation) {
getAnnotators().forEach(a -> a.recordAnnotation(annotation));
}
// public static synchronized void recordAnnotation(
// String sessionName,
// long startEpochMillis,
// long endEpochMillis,
// Map<String, String> target,
// Map<String, String> details) {
// getAnnotators().forEach(a -> a.recordAnnotation(sessionName, startEpochMillis, endEpochMillis, target, details));
// }
// public static synchronized void recordAnnotation(
// String sessionName,
// Map<String, String> target,
// Map<String, String> details) {
// recordAnnotation(sessionName, 0L, 0L, target, details);
// }
}

15
engine-core/src/main/java/io/nosqlbench/engine/core/metrics/OnError.java Normal file
View File

@ -0,0 +1,15 @@
package io.nosqlbench.engine.core.metrics;
public enum OnError {
Warn,
Throw;
public static OnError valueOfName(String name) {
for (OnError value : OnError.values()) {
if (value.toString().toLowerCase().equals(name.toLowerCase())) {
return value;
}
}
throw new RuntimeException("No matching OnError enum value for '" + name + "'");
}
}

13
engine-core/src/main/java/io/nosqlbench/engine/core/script/Scenario.java
View File

@ -231,9 +231,18 @@ public class Scenario implements Callable<ScenarioResult> {
}
public void run() {
state=State.Running;
state = State.Running;
startedAtMillis=System.currentTimeMillis();
startedAtMillis = System.currentTimeMillis();
Annotators.recordAnnotation(
Annotation.newBuilder()
.session(this.scenarioName)
.now()
.layer(Layer.Scenario)
.label("scenario", getScenarioName())
.detail("engine", this.engine.toString())
.build()
);
init();
logger.debug("Running control script for " + getScenarioName() + ".");
for (String script : scripts) {

38
nb-api/src/main/java/io/nosqlbench/nb/api/Layer.java Normal file
View File

@ -0,0 +1,38 @@
package io.nosqlbench.nb.api;
public enum Layer {
/**
* Events which describe command line arguments, such as parsing,
* named scenario mapping, or critical errors
*/
CLI,
/**
* Events which describe scenario execution, such as parameters,
* lifecycle events, and critical errors
*/
Scenario,
/**
* Events which describe scripting details, such as extensions,
* sending programmatic annotations, or critical errors
*/
Script,
/**
* Events which are associated with a particular activity instance,
* such as parameters, starting and stopping, and critical errors
*/
Activity,
/**
* Events which are associated with a particular activity thread
*/
Motor,
/**
* Events which are associated with a particular operation or op template
*/
Operation
}

43
nb-api/src/main/java/io/nosqlbench/nb/api/annotation/Annotator.java
View File

@ -1,43 +0,0 @@
package io.nosqlbench.nb.api.annotation;
import io.nosqlbench.nb.spi.Named;
import java.util.Map;
/**
* An implementation of this type is responsible for taking annotation details and
* logging them in a useful place.
*/
public interface Annotator extends Named {
/**
* Submit an annotation to some type of annotation store or logging or eventing mechanism.
* Implementations of this service are responsible for mapping the scenarioName, target,
* and details into the native schema of the target annotation or logging system in whichever
* way would be the least surprising for a user.
*
* The target is the nominative data which identifies the identity of the annotation. This
* must include enough information to allow the annotation to be homed and located within
* a target system such that is is visible where it should be seen. This includes all
* metadata which may be used to filter or locate the annotation, including timestamps.
*
* The details contain payload information to be displayed within the body of the annotation.
*
* @param sessionName The name of the scenario
* @param startEpochMillis The epoch millisecond instant of the annotation, set this to 0 to have it
* automatically set to the current system time.
* @param endEpochMillis The epoch millisecond instant at the end of the interval. If this is
* equal to the start instant, then this is an annotation for a point in time.
* This will be the default behavior if this value is 0.
* @param target The target of the annotation, fields which are required to associate the
* annotation with the correct instance of a dashboard, metrics, etc
* @param details A map of details
*/
void recordAnnotation(
String sessionName,
long startEpochMillis,
long endEpochMillis,
Map<String, String> target,
Map<String, String> details);
}

76
nb-api/src/main/java/io/nosqlbench/nb/api/annotations/Annotation.java Normal file
View File

@ -0,0 +1,76 @@
package io.nosqlbench.nb.api.annotations;
import io.nosqlbench.nb.api.Layer;
import java.util.Map;
/**
* This is a general purpose representation of an event that describes
* a significant workflow detail to users running tests. It can be
* an event that describes an instant, or it can describe an interval
* in time (being associated with the interval of time between two
* canonical events.)
*
* This view of an annotation event captures the semantics of what
* any reportable annotation should look like from the perspective of
* NoSQLBench. It is up to the downstream consumers to map these
* to concrete fields or identifiers as appropriate.
*/
public interface Annotation {
/**
* @return The named session that the annotation is associated with
*/
String getSession();
/**
* If this is the same as {@link #getEnd()}, then the annotation is
* for an instant in time.
*
* @return The beginning of the interval of time that the annotation describes
*/
long getStart();
/**
* If this is the same as {@link #getStart()}, then the annotation
* is for an instant in time.
*
* @return The end of the interval of time that the annotation describes
*/
long getEnd();
/**
* Annotations must be associated with a processing layer in NoSQLBench.
* For more details on layers, see {@link Layer}
*
* @return
*/
Layer getLayer();
/**
* The labels which identify what this annotation pertains to. The following labels
* should be provided for every annotation, when available:
* <UL>
* <LI>appname: "nosqlbench"</LI>
* <LI>alias: The name of the activity alias, if available</LI>
* <LI>workload: The name of the workload file, if named scenarios are used</LI>
* <LI>scenario: The name of the named scenario, if named scenarios are used</LI>
* <LI>step: The name of the named scenario step, if named scenario are used</LI>
* <LI>usermode: "named_scenario" or "adhoc_activity"</LI>
* </UL>
*
* @return The labels map
*/
Map<String, String> getLabels();
/**
* The details are an ordered map of all the content that you would want the user to see.
*
* @return The details map
*/
Map<String, String> getDetails();
static BuilderFacets.WantsSession newBuilder() {
return new AnnotationBuilder();
}
}

77
nb-api/src/main/java/io/nosqlbench/nb/api/annotations/AnnotationBuilder.java Normal file
View File

@ -0,0 +1,77 @@
package io.nosqlbench.nb.api.annotations;
import io.nosqlbench.nb.api.Layer;
import java.util.LinkedHashMap;
public class AnnotationBuilder implements BuilderFacets.All {
private String session;
private long start;
private long end;
private final LinkedHashMap<String, String> labels = new LinkedHashMap<>();
private final LinkedHashMap<String, String> details = new LinkedHashMap<>();
private Layer layer;
@Override
public AnnotationBuilder layer(Layer layer) {
this.layer = layer;
this.label("layer", layer.toString());
return this;
}
@Override
public AnnotationBuilder interval(long start, long end) {
start(start);
end(end);
return this;
}
@Override
public AnnotationBuilder now() {
start(System.currentTimeMillis());
end(this.start);
return this;
}
private AnnotationBuilder start(long start) {
this.start = start;
return this;
}
private AnnotationBuilder end(long end) {
this.end = end;
return this;
}
@Override
public AnnotationBuilder at(long at) {
this.start(at);
this.end(at);
return this;
}
@Override
public AnnotationBuilder label(String name, String value) {
this.labels.put(name, value);
return this;
}
@Override
public BuilderFacets.WantsMoreDetailsOrBuild detail(String name, String value) {
this.details.put(name, value);
return this;
}
@Override
public Annotation build() {
return new MutableAnnotation(session, layer, start, end, labels, details).asReadOnly();
}
@Override
public BuilderFacets.WantsInterval session(String session) {
this.session = session;
return this;
}
}

21
nb-api/src/main/java/io/nosqlbench/nb/api/annotations/Annotator.java Normal file
View File

@ -0,0 +1,21 @@
package io.nosqlbench.nb.api.annotations;
import io.nosqlbench.nb.spi.Named;
/**
* An implementation of this type is responsible for taking annotation details and
* logging them in a useful place.
*/
public interface Annotator extends Named {
/**
* Submit an annotation to some type of annotation store, logging or eventing mechanism.
* Implementations of this service are responsible for mapping the scenario and labels
* into appropriate key data, and the details in to a native payload. The least surprising
* and most obvious mapping should be used in each case.
*
* For details on constructing a useful annotation to submit to this service, see {@link Annotation#newBuilder()}
*/
void recordAnnotation(Annotation annotation);
}

58
nb-api/src/main/java/io/nosqlbench/nb/api/annotations/BuilderFacets.java Normal file
View File

@ -0,0 +1,58 @@
package io.nosqlbench.nb.api.annotations;
import io.nosqlbench.nb.api.Layer;
public interface BuilderFacets {
interface All extends
WantsSession, WantsInterval, WantsLayer, WantsLabels, WantsMoreDetailsOrBuild, WantsMoreLabelsOrDetails {
}
interface WantsSession {
/**
* The session is the global name of a NoSQLBench process which run a scenario. It is required.
*/
WantsInterval session(String session);
}
interface WantsInterval {
/**
* Specify the instant of the annotated event.
*
* @param epochMillis
*/
WantsLayer at(long epochMillis);
/**
* An interval annotation spans the time between two instants.
*/
WantsLayer interval(long startMillis, long endMillis);
/**
* Use the current UTC time as the annotation instant.
*/
WantsLayer now();
}
interface WantsLayer {
WantsMoreLabelsOrDetails layer(Layer layer);
}
interface WantsLabels {
WantsMoreLabelsOrDetails label(String name, String value);
}
interface WantsMoreLabelsOrDetails {
WantsMoreLabelsOrDetails label(String name, String value);
WantsMoreDetailsOrBuild detail(String name, String value);
}
interface WantsMoreDetailsOrBuild {
WantsMoreDetailsOrBuild detail(String name, String value);
Annotation build();
}
}

121
nb-api/src/main/java/io/nosqlbench/nb/api/annotations/MutableAnnotation.java Normal file
View File

@ -0,0 +1,121 @@
package io.nosqlbench.nb.api.annotations;
import io.nosqlbench.nb.api.Layer;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Map;
public class MutableAnnotation implements Annotation {
private String session = "SESSION_UNNAMED";
private Layer layer;
private long start = 0L;
private long end = 0L;
private Map<String, String> labels = new LinkedHashMap<>();
private Map<String, String> details = new LinkedHashMap<>();
public MutableAnnotation(String session, Layer layer, long start, long end, LinkedHashMap<String, String> labels,
LinkedHashMap<String, String> details) {
this.session = session;
this.layer = layer;
this.start = start;
this.end = end;
this.details = details;
this.labels = labels;
}
public void setSession(String sessionName) {
this.session = sessionName;
}
public void setStart(long intervalStart) {
this.start = intervalStart;
}
public void setEnd(long intervalEnd) {
this.end = intervalEnd;
}
public void setLabels(Map<String, String> labels) {
this.labels = labels;
}
public void setLayer(Layer layer) {
this.layer = layer;
this.labels.put("layer", layer.toString());
}
public void setDetails(Map<String, String> details) {
this.details = details;
}
@Override
public String getSession() {
return session;
}
@Override
public long getStart() {
return start;
}
@Override
public long getEnd() {
return end;
}
@Override
public Layer getLayer() {
return this.layer;
}
@Override
public Map<String, String> getLabels() {
return labels;
}
@Override
public Map<String, String> getDetails() {
return details;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("session: ").append(getSession()).append("\n");
sb.append("[").append(new Date(getStart()));
if (getStart() != getEnd()) {
sb.append(" - ").append(new Date(getEnd()));
}
sb.append("]\n");
sb.append("details:\n");
formatMap(sb, getDetails());
sb.append("labels:\n");
formatMap(sb, getLabels());
return sb.toString();
}
private void formatMap(StringBuilder sb, Map<String, String> details) {
details.forEach((k, v) -> {
sb.append(" ").append(k).append(": ");
if (v.contains("\n")) {
sb.append("\n");
String[] lines = v.split("\n+");
for (String line : lines) {
sb.append(" " + line + "\n");
}
// Arrays.stream(lines).sequential().map(s -> " "+s+"\n").forEach(sb::append);
} else {
sb.append(v).append("\n");
}
});
}
public Annotation asReadOnly() {
return this;
}
}

45
nb-api/src/test/java/io/nosqlbench/nb/api/annotations/AnnotationBuilderTest.java Normal file
View File

@ -0,0 +1,45 @@
package io.nosqlbench.nb.api.annotations;
import io.nosqlbench.nb.api.Layer;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class AnnotationBuilderTest {
private static final long time = 1600000000000L;
@Test
public void testBasicAnnotation() {
Annotation an1 = Annotation.newBuilder()
.session("test-session")
.at(time)
.layer(Layer.Scenario)
.label("labelka", "labelvb")
.label("labelkc", "labelvd")
.detail("detailk1", "detailv1")
.detail("detailk2", "detailv21\ndetailv22")
.detail("detailk3", "v1\nv2\nv3\n")
.build();
String represented = an1.toString();
assertThat(represented).isEqualTo("session: test-session\n" +
"[Sun Sep 13 07:26:40 CDT 2020]\n" +
"details:\n" +
" detailk1: detailv1\n" +
" detailk2: \n" +
" detailv21\n" +
" detailv22\n" +
" detailk3: \n" +
" v1\n" +
" v2\n" +
" v3\n" +
"labels:\n" +
" layer: Scenario\n" +
" labelka: labelvb\n" +
" labelkc: labelvd\n");
}
}