IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'jshook/heartbeat'

Browse Source
This commit is contained in:
Jonathan Shook 2024-01-09 15:06:50 -06:00
commit ed24a67916
18 changed files with 685 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -95,7 +95,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
* for scenario encapsulation and concurrent testing.
*
* @param args
* Command Line Args
* Command Line Args
*/
public static void main(final String[] args) {
try {
@ -158,17 +158,17 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
this.sessionName = SessionNamer.format(globalOptions.getSessionName(), sessionTime).replaceAll("SESSIONCODE", sessionCode);
NBCLI.loggerConfig
.setSessionName(sessionName)
.setConsoleLevel(globalOptions.getConsoleLogLevel())
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
.setLogfileLevel(globalOptions.getScenarioLogLevel())
.setLogfilePattern(globalOptions.getLogfileLoggingPattern())
.setLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
.setMaxLogs(globalOptions.getLogsMax())
.setLogsDirectory(globalOptions.getLogsDirectory())
.setAnsiEnabled(globalOptions.isEnableAnsi())
.setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger())
.activate();
.setSessionName(sessionName)
.setConsoleLevel(globalOptions.getConsoleLogLevel())
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
.setLogfileLevel(globalOptions.getScenarioLogLevel())
.setLogfilePattern(globalOptions.getLogfileLoggingPattern())
.setLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
.setMaxLogs(globalOptions.getLogsMax())
.setLogsDirectory(globalOptions.getLogsDirectory())
.setAnsiEnabled(globalOptions.isEnableAnsi())
.setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger())
.activate();
ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig);
NBCLI.logger = LogManager.getLogger("NBCLI");
@ -208,8 +208,8 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
if (annotatorsConfig == null || annotatorsConfig.isBlank()) {
List<Map<String, String>> annotatorsConfigs = new ArrayList<>();
annotatorsConfigs.add(Map.of(
"type", "log",
"level", "info"
"type", "log",
"level", "info"
));
Gson gson = new GsonBuilder().create();
@ -277,18 +277,18 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
NBCLI.logger.debug(() -> "user requests to cat " + resourceToCat);
Optional<Content<?>> tocat = NBIO.classpath()
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCat).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCat).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocat.isEmpty()) tocat = NBIO.classpath()
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCat).first();
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCat).first();
final Content<?> data = tocat.orElseThrow(
() -> new BasicError("Unable to find " + resourceToCat +
" in classpath to cat out"));
() -> new BasicError("Unable to find " + resourceToCat +
" in classpath to cat out"));
System.out.println(data.get());
NBCLI.logger.info(() -> "Dumped internal resource '" + data.asPath() + "' to stdout");
@ -300,19 +300,19 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
NBCLI.logger.debug(() -> "user requests to copy out " + resourceToCopy);
Optional<Content<?>> tocopy = NBIO.classpath()
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocopy.isEmpty()) tocopy = NBIO.classpath()
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).first();
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).first();
final Content<?> data = tocopy.orElseThrow(
() -> new BasicError(
"Unable to find " + resourceToCopy +
" in classpath to copy out")
() -> new BasicError(
"Unable to find " + resourceToCopy +
" in classpath to copy out")
);
final Path writeTo = Path.of(data.asPath().getFileName().toString());
@ -350,7 +350,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
if (options.wantsTopicalHelp()) {
final Optional<String> helpDoc = MarkdownFinder.forHelpTopic(options.wantsTopicalHelpFor());
System.out.println(helpDoc.orElseThrow(
() -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
() -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
));
return NBCLI.EXIT_OK;
}
@ -358,12 +358,12 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
NBCLI.logger.debug("initializing annotators with config:'{}'", annotatorsConfig);
Annotators.init(annotatorsConfig, options.getAnnotateLabelSpec());
Annotators.recordAnnotation(
Annotation.newBuilder()
.element(this)
.now()
.layer(Layer.Session)
.addDetail("cli", String.join("\n", args))
.build()
Annotation.newBuilder()
.element(this)
.now()
.layer(Layer.Session)
.addDetail("cli", String.join("\n", args))
.build()
);
// if ((null != reportPromPushTo) || (null != reportGraphiteTo) || (null != options.wantsReportCsvTo())) {
@ -401,54 +401,58 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
// intentionally not shown for warn-only
NBCLI.logger.info(() -> "console logging level is " + options.getConsoleLogLevel());
Map<String, String> props = Map.of(
"summary", options.getReportSummaryTo(),
"logsdir", options.getLogsDirectory().toString(),
"progress", options.getProgressSpec(),
"prompush_cache", "prompush_cache.txt",
"heartbeat", String.valueOf(options.wantsHeartbeatIntervalMs())
);
/**
* At this point, the command stream from the CLI should be handed into the session, and the session should
* marshal and transform it for any scenario invocations directly.
*/
NBSession session = new NBSession(
new NBBaseComponent(null,
options.getLabelMap()
.andDefault("jobname", "nosqlbench")
.andDefault("instance", "default")
),
sessionName
);
// TODO: Decide whether this should be part of ctor consistency
Map.of(
"summary", options.getReportSummaryTo(),
"logsdir", options.getLogsDirectory().toString(),
"progress", options.getProgressSpec(),
"prompush_cache", "prompush_cache.txt"
).forEach(session::setComponentProp);
options.wantsReportCsvTo().ifPresent(cfg -> {
MetricInstanceFilter filter = new MetricInstanceFilter();
filter.addPattern(cfg.pattern);
new CsvReporter(session, Path.of(cfg.file), cfg.millis, filter);
});
try (
NBSession session = new NBSession(
new NBBaseComponent(null,
options.getLabelMap()
.andDefault("jobname", "nosqlbench")
.andDefault("instance", "default")
),
sessionName,
props
)) {
options.wantsReportPromPushTo().ifPresent(cfg -> {
String[] words = cfg.split(",");
String uri;
long intervalMs = 10_000L;
options.wantsReportCsvTo().ifPresent(cfg -> {
MetricInstanceFilter filter = new MetricInstanceFilter();
filter.addPattern(cfg.pattern);
new CsvReporter(session, Path.of(cfg.file), cfg.millis, filter);
});
switch (words.length) {
case 2:
intervalMs = Unit.msFor(words[1]).orElseThrow(() -> new RuntimeException("can't parse '" + words[1] + "!"));
case 1:
uri = words[0];
break;
default:
throw new RuntimeException("Unable to parse '" + cfg + "', must be in <URI> or <URI>,ms form");
}
session.create().pushReporter(uri, intervalMs, NBLabels.forKV());
});
options.wantsReportPromPushTo().ifPresent(cfg -> {
String[] words = cfg.split(",");
String uri;
long intervalMs = 10_000L;
switch (words.length) {
case 2:
intervalMs = Unit.msFor(words[1]).orElseThrow(() -> new RuntimeException("can't parse '" + words[1] + "!"));
case 1:
uri = words[0];
break;
default:
throw new RuntimeException("Unable to parse '" + cfg + "', must be in <URI> or <URI>,ms form");
}
session.create().pushReporter(uri, intervalMs, NBLabels.forKV());
});
ExecutionResult sessionResult = session.apply(options.getCommands());
ExecutionResult sessionResult = session.apply(options.getCommands());
logger.info(sessionResult);
return sessionResult.getStatus().code;
}
// sessionResult.printSummary(System.out);
logger.info(sessionResult);
return sessionResult.getStatus().code;
}
@ -465,10 +469,8 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
}
basicHelp = basicHelp.replaceAll("PROG", this.commandName);
return basicHelp;
}
@Override
public NBLabels getLabels() {
return labels;

12
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -63,6 +63,8 @@ public class NBCLIOptions {
private static final String LABELSPEC = "--labelspec";
private static final String ANNOTATORS_CONFIG = "--annotators";
private static final String HEARTBEAT_MILLIS = "--heartbeat-millis";
// Enabled if the TERM env var is provided
private static final String ANSI = "--ansi";
@ -202,6 +204,7 @@ public class NBCLIOptions {
private String annotateLabelSpec = "";
private String metricsLabelSpec = "";
private String wantsToCatResource ="";
private long heartbeatIntervalMs;
public boolean wantsLoggedMetrics() {
return this.wantsConsoleMetrics;
@ -642,6 +645,11 @@ public class NBCLIOptions {
arglist.removeFirst();
this.wantsToCatResource = this.readWordOrThrow(arglist, "workload to cat");
break;
case HEARTBEAT_MILLIS:
arglist.removeFirst();
this.heartbeatIntervalMs =
Long.parseLong(this.readWordOrThrow(arglist, "heartbeat interval in ms"));
break;
default:
nonincludes.addLast(arglist.removeFirst());
}
@ -739,6 +747,10 @@ public class NBCLIOptions {
return this.wantsActivityTypes;
}
public long wantsHeartbeatIntervalMs() {
return this.heartbeatIntervalMs;
}
public boolean wantsTopicalHelp() {
return this.wantsActivityHelp;
}

24
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java
View File

@ -16,6 +16,8 @@
package io.nosqlbench.engine.core.lifecycle.session;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
import io.nosqlbench.nb.api.components.status.NBLiveComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBFunctionGauge;
@ -43,7 +45,7 @@ import java.util.function.Function;
* on.
* All NBScenarios are run within an NBSession.
*/
public class NBSession extends NBBaseComponent implements Function<List<Cmd>, ExecutionResult>, NBTokenWords {
public class NBSession extends NBLiveComponent implements Function<List<Cmd>, ExecutionResult>, NBTokenWords {
private final static Logger logger = LogManager.getLogger(NBSession.class);
// private final ClientSystemMetricChecker clientMetricChecker;
@ -57,19 +59,22 @@ public class NBSession extends NBBaseComponent implements Function<List<Cmd>, Ex
public NBSession(
NBLabeledElement labelContext,
String sessionName
String sessionName,
Map<String, String> props
) {
super(
null,
labelContext.getLabels()
.and("session", sessionName)
.and("session", sessionName),
props,
"session"
);
new NBSessionSafetyMetrics(this);
create().gauge(
"session_time",
() -> (double)System.nanoTime(),
() -> (double) System.nanoTime(),
MetricCategory.Core,
"session time in nanoseconds"
);
@ -86,13 +91,16 @@ public class NBSession extends NBBaseComponent implements Function<List<Cmd>, Ex
ResultCollector collector = new ResultCollector();
try (ResultContext results = new ResultContext(collector).ok()) {
for (NBCommandAssembly.CommandInvocation invocation : invocationCalls) {
try {
String targetContext = invocation.containerName();
String targetContext = invocation.containerName();
String explanation = "in context '" + targetContext + "'";
try (NBInvokableCommand command = invocation.command()) {
explanation += " command '" + command.toString() + "'";
NBBufferedContainer container = getContext(targetContext);
NBCommandResult cmdResult = container.apply(invocation.command(), invocation.params());
NBCommandResult cmdResult = container.apply(command, invocation.params());
results.apply(cmdResult);
} catch (Exception e) {
String msg = "While running command '" + invocation.command() + "' in container '" + invocation.containerName() + "', an error occurred: " + e.toString();
String msg = "While running " + explanation + "', an error occurred: " + e.toString();
onError(e);
logger.error(msg);
results.error(e);
break;

2
engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java
View File

@ -391,7 +391,7 @@ public class LoggerConfig extends ConfigurationFactory {
private void attachAuxLogger(ConfigurationBuilder<BuiltConfiguration> builder, String loggerName, Level fileLevel) {
String appenderName = loggerName+(("_APPENDER").toUpperCase());
String fileName = loggerDir.resolve(getFileBase() + "_"+loggerName+".log").toString().toLowerCase();
String fileName = loggerDir.resolve(getFileBase() + "_"+loggerName.toLowerCase()+".log").toString();
var appender = builder
.newAppender(appenderName, FileAppender.PLUGIN_NAME)
.addAttribute("append", false)

63
nb-api/src/main/java/io/nosqlbench/nb/api/components/core/NBBaseComponent.java
View File

@ -21,7 +21,6 @@ import io.nosqlbench.nb.api.components.events.ComponentOutOfScope;
import io.nosqlbench.nb.api.components.events.DownEvent;
import io.nosqlbench.nb.api.components.events.NBEvent;
import io.nosqlbench.nb.api.components.events.UpEvent;
import io.nosqlbench.nb.api.config.params.ElementData;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetric;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.LogManager;
@ -30,30 +29,37 @@ import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
public class NBBaseComponent extends NBBaseComponentMetrics implements NBComponent, NBTokenWords {
public class NBBaseComponent extends NBBaseComponentMetrics implements NBComponent, NBTokenWords, NBComponentTimeline {
private final static Logger logger = LogManager.getLogger("RUNTIME");
protected final NBComponent parent;
protected final NBLabels labels;
private final List<NBComponent> children = new ArrayList<>();
private long endAt=0L;
private final long startAt;
protected NBMetricsBuffer metricsBuffer = new NBMetricsBuffer();
protected boolean bufferOrphanedMetrics = false;
private ConcurrentHashMap<String,String> props = new ConcurrentHashMap<>();
protected Exception error;
protected long started_ns, teardown_ns, closed_ns, errored_ns;
protected NBInvokableState state = NBInvokableState.STARTING;
public NBBaseComponent(NBComponent parentComponent) {
this(parentComponent, NBLabels.forKV());
}
public NBBaseComponent(NBComponent parentComponent, NBLabels componentSpecificLabelsOnly) {
this.started_ns = System.nanoTime();
this.labels = componentSpecificLabelsOnly;
this.startAt = System.nanoTime();
if (parentComponent != null) {
parent = parentComponent;
parent.attachChild(this);
} else {
parent = null;
}
state = (state==NBInvokableState.ERRORED) ? state : NBInvokableState.RUNNING;
}
public NBBaseComponent(NBComponent parentComponent, NBLabels componentSpecificLabelsOnly, Map<String, String> props) {
this(parentComponent,componentSpecificLabelsOnly);
props.forEach(this::setComponentProp);
}
@Override
@ -117,30 +123,40 @@ public class NBBaseComponent extends NBBaseComponentMetrics implements NBCompone
@Override
public final void close() throws RuntimeException {
state = (state==NBInvokableState.ERRORED) ? state : NBInvokableState.CLOSING;
closed_ns = System.nanoTime();
try {
logger.debug("cleaning up");
ArrayList<NBComponent> children = new ArrayList<>(getChildren());
for (NBComponent child : children) {
child.close();
}
teardown();
} catch (Exception e) {
logger.error(e);
onError(e);
} finally {
logger.debug("detaching " + description());
if (parent != null) {
parent.detachChild(this);
}
teardown();
}
}
public void onError(Exception e) {
RuntimeException wrapped = new RuntimeException("While in state " + this.state + ", an error occured: " + e, e);
logger.error(wrapped);
this.error = wrapped;
state=NBInvokableState.ERRORED;
}
/**
* Override this method in your component implementations when you need to do something
* to close out your component.
*/
protected void teardown() {
logger.debug("tearing down " + description());
this.endAt = System.nanoTime();
this.teardown_ns = System.nanoTime();
this.state=(state==NBInvokableState.ERRORED) ? state : NBInvokableState.STOPPED;
}
@Override
@ -229,10 +245,10 @@ public class NBBaseComponent extends NBBaseComponentMetrics implements NBCompone
@Override
public long getNanosSinceStart() {
if (endAt==0) {
return System.nanoTime()-startAt;
if (teardown_ns ==0) {
return System.nanoTime()- started_ns;
} else {
return endAt-startAt;
return teardown_ns - started_ns;
}
}
@ -255,4 +271,29 @@ public class NBBaseComponent extends NBBaseComponentMetrics implements NBCompone
props.put(name, value);
return this;
}
@Override
public NBInvokableState getComponentState() {
return state;
}
@Override
public long nanosof_start() {
return this.started_ns;
}
@Override
public long nanosof_close() {
return this.closed_ns;
}
@Override
public long nanosof_teardown() {
return this.teardown_ns;
}
@Override
public long nanosof_error() {
return this.errored_ns;
}
}

50
nb-api/src/main/java/io/nosqlbench/nb/api/components/core/NBComponentTimeline.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.core;
public interface NBComponentTimeline {
NBInvokableState getComponentState();
/**
* This will be 0L if the component hasn't fully started, else it will be
* the {@link System#nanoTime()} of when the component entered its constructor
* @return nanosecond time of component construction
*/
long nanosof_start();
/**
* This will be 0L if the component hasn't began the process of closing down.
* @return nanosecond time of invoking {@link NBBaseComponent#close()}
*/
long nanosof_close();
/**
* This will be 0L if the component hasn't completed teardown. Otherwise it will be
* the {@link System#nanoTime()} when the base teardown logic in the component has completed.
* For this reason, it is imperative that any overrides to {@link NBBaseComponent#teardown()}
* are called, and called last in the overridden teardown method.
* @return nanosecond time of teardown completion
*/
long nanosof_teardown();
/**
* This will be 0L if the component hasn't logged an error. Otherwise it will be
* the {@link System#nanoTime()} of when the error was reported.
* @return nanosecond time of the error
*/
long nanosof_error();
}

59
nb-api/src/main/java/io/nosqlbench/nb/api/components/core/NBInvokableState.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.core;
/**
* <PRE>{@code
* errored_at > 0 -> ERROR
* started_at == 0 -> STARTING
* <p>
* <p>
* <p>
* started_at > closed_at
* STARTING
* closed_at > started_at
* RUNNING
* teardown_at > closed_at
* STOPPING
* teardown_at
* STOPPED
* stopped_at
* }</pre>
*/
public enum NBInvokableState {
/**
* The component exists in some state but has not completed initialization / construction
*/
STARTING,
/**
* The component has completed initialization and is presumed to be running
*/
RUNNING,
/**
* The component has begun closing down, which means unwinding/closing any child components
*/
CLOSING,
/**
* The component has completed closing down, including its teardown logic
*/
STOPPED,
/**
* There was an error
*/
ERRORED;
}

88
nb-api/src/main/java/io/nosqlbench/nb/api/components/core/PeriodicTaskComponent.java
View File

@ -25,90 +25,10 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public abstract class PeriodicTaskComponent extends NBBaseComponent implements Runnable {
public abstract class PeriodicTaskComponent extends UnstartedPeriodicTaskComponent {
private static final Logger logger = LogManager.getLogger(PeriodicTaskComponent.class);
private final long intervalmillis;
private final Lock lock = new ReentrantLock();
private final Condition shutdownSignal = lock.newCondition();
private final boolean oneLastTime;
Thread thread;
private boolean running = true;
public PeriodicTaskComponent(
NBComponent node,
NBLabels extraLabels,
long millis,
boolean oneLastTime,
String threadName
) {
super(node, extraLabels);
this.intervalmillis = millis;
thread = Thread.ofVirtual().name(threadName).start(this);
this.oneLastTime = oneLastTime;
public PeriodicTaskComponent(NBComponent node, NBLabels extraLabels, long millis, String threadName, FirstReport firstReport, LastReport lastReport) {
super(node, extraLabels, millis, threadName, firstReport, lastReport);
start();
}
protected abstract void task();
@Override
public void run() {
long now = System.currentTimeMillis();
long reportAt = now + intervalmillis;
long waitfor = reportAt - now;
while (running) {
while (running && waitfor > 0) {
boolean signalReceived = false;
try {
lock.lock();
signalReceived = shutdownSignal.await(waitfor, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {
} finally {
lock.unlock();
}
if (signalReceived) {
logger.debug("signal shutting down " + this);
return;
}
now = System.currentTimeMillis();
waitfor = reportAt - now;
}
// logger.info("summarizing metrics to console");
try {
task();
} catch (Exception e) {
logger.error(e);
throw new RuntimeException(e);
} finally {
reportAt = reportAt + (intervalmillis);
now = System.currentTimeMillis();
waitfor = reportAt - now;
}
}
logger.info("shutting down periodic runnable component: " + description());
}
public void teardown() {
logger.debug("shutting down " + this);
lock.lock();
running = false;
shutdownSignal.signalAll();
lock.unlock();
logger.debug("signaled reporter thread to shut down " + description());
try {
thread.join();
} catch (InterruptedException e) {
logger.warn("interrupted while joining thread");
}
if (oneLastTime) {
logger.debug("running " + this + " one last time.");
task();
}
}
}

167
nb-api/src/main/java/io/nosqlbench/nb/api/components/core/UnstartedPeriodicTaskComponent.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.core;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* <P>Because of ctor super calling order requirements, the task thread can't always be started
* automatically in super(...). If that is the case, then use this class directly and call
* start() at the end of your subtype ctor.</P>
*
* <P>Otherwise, it is safe to use {@link PeriodicTaskComponent} directly.</P>
*/
public abstract class UnstartedPeriodicTaskComponent extends NBBaseComponent implements Runnable {
private static final Logger logger = LogManager.getLogger(UnstartedPeriodicTaskComponent.class);
protected final long intervalmillis;
private final Lock lock = new ReentrantLock();
private final Condition shutdownSignal = lock.newCondition();
private final FirstReport firstReport;
private final LastReport lastReport;
private final String threadName;
Thread thread;
private boolean running = true;
public enum FirstReport {
Immediately,
OnInterval
}
public enum LastReport {
None,
onClose,
/**
* OnInterrupt is a stronger version of OnClose, including scenarios where the process is interrupted with a signal
*/
OnInterrupt
}
public UnstartedPeriodicTaskComponent(
NBComponent node,
NBLabels extraLabels,
long millis,
String threadName,
FirstReport firstReport,
LastReport lastReport
) {
super(node, extraLabels);
this.threadName = threadName;
this.intervalmillis = millis;
this.firstReport = firstReport;
this.lastReport = lastReport;
if(lastReport== LastReport.OnInterrupt) {
Thread hook=new Thread(this::task,"shutdownhook-"+threadName);
Runtime.getRuntime().addShutdownHook(hook);
}
// TODO: There is a potential race condition between init and invoke here, if millis is low enough and post-super() state is needed
}
public void start() {
if (firstReport==FirstReport.Immediately) task();
thread = Thread.ofVirtual().name(threadName).start(this);
}
/**
* This task should only do what is needed once each period.
* If it throws any exceptions, then these exceptions will cause the period task
* to exit. Thus, if you need to allow failures in some cases while keeping
* the caller (scheduler) active, all errors should be caught and handled
* internally.
*/
protected abstract void task();
@Override
public void run() {
long now = System.currentTimeMillis();
long reportAt = now + intervalmillis;
long waitfor = reportAt - now;
while (running) {
while (running && waitfor > 0) {
boolean signalReceived = false;
try {
lock.lock();
signalReceived = shutdownSignal.await(waitfor, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {
} finally {
lock.unlock();
}
if (signalReceived) {
logger.debug("signal shutting down " + this);
return;
}
now = System.currentTimeMillis();
waitfor = reportAt - now;
}
// logger.info("summarizing metrics to console");
try {
lock.lock();
task();
} catch (Exception e) {
logger.error(e);
throw new RuntimeException(e);
} finally {
reportAt = reportAt + (intervalmillis);
now = System.currentTimeMillis();
waitfor = reportAt - now;
lock.unlock();
}
}
logger.info("shutting down periodic runnable component: " + description());
}
public void teardown() {
logger.debug("shutting down " + this);
lock.lock();
running = false;
shutdownSignal.signalAll();
lock.unlock();
// if (lastReport==LastReport.onClose || lastReport==LastReport.OnInterrupt) {
// logger.debug("final task() call for period component " + description());
// task();
// }
logger.debug("signaled reporter thread to shut down " + description());
try {
thread.join();
} catch (InterruptedException e) {
logger.warn("interrupted while joining thread");
}
if (this.lastReport== LastReport.onClose) {
logger.debug("running " + this + " one last time on close().");
task();
}
super.teardown();
}
}

59
nb-api/src/main/java/io/nosqlbench/nb/api/components/status/ComponentPulse.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.status;
import io.nosqlbench.nb.api.components.core.UnstartedPeriodicTaskComponent;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
public class ComponentPulse extends UnstartedPeriodicTaskComponent {
private final static Logger logger = LogManager.getLogger(ComponentPulse.class);
private final Path hbpath;
private final NBLiveComponent pulseOf;
public ComponentPulse(NBLiveComponent pulseOf, NBLabels extraLabels, String fileNameLabel, long millis) {
super(
pulseOf,
extraLabels,
millis,
"PULSE-" + pulseOf.description(),
FirstReport.Immediately,
LastReport.OnInterrupt
);
this.pulseOf = pulseOf;
String logsdir = getComponentProp("logsdir").orElseThrow();
this.hbpath = Path.of(logsdir).resolve(pulseOf.getLabels().valueOf(fileNameLabel)+"_status.yaml");
start();
}
@Override
protected void task() {
logger.debug("emitting pulse for :" + this.pulseOf.description());
Heartbeat heartbeat = pulseOf.heartbeat().withHeartbeatDetails(intervalmillis,System.currentTimeMillis());
try {
Files.writeString(hbpath, heartbeat.toYaml(), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
} catch (IOException e) {
logger.error("Unable to write heartbeat data to " + hbpath.toString() + ": " + e);
}
}
}

73
nb-api/src/main/java/io/nosqlbench/nb/api/components/status/Heartbeat.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.status;
import io.nosqlbench.nb.api.components.core.NBInvokableState;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.snakeyaml.engine.v2.api.Dump;
import org.snakeyaml.engine.v2.api.DumpSettings;
import org.snakeyaml.engine.v2.common.FlowStyle;
import java.util.Map;
public record Heartbeat(
NBLabels labels,
NBInvokableState state,
long started_at,
long session_time_ns,
long heartbeat_interval_ms,
long heartbeat_epoch_ms
) {
public final static Dump dump = createDump();
private static Dump createDump() {
DumpSettings settings = DumpSettings.builder().setDefaultFlowStyle(FlowStyle.BLOCK).build();
return new Dump(settings, new HeartbeatRepresenter(settings));
}
public Heartbeat withHeartbeatDetails(long new_heartbeat_interval_ms, long new_heartbeat_ms_epoch) {
return new Heartbeat(
labels,
state,
started_at,
session_time_ns,
new_heartbeat_interval_ms,
new_heartbeat_ms_epoch
);
}
public String toYaml() {
return toString();
}
public Map<String, Object> toMap() {
return Map.of(
"labels", labels.asMap(),
"state", state,
"started_at_epochms", started_at,
"session_time_ns", session_time_ns,
"heartbeat_interval_ms", heartbeat_interval_ms,
"heartbeat_epoch_ms", heartbeat_epoch_ms
);
}
@Override
public String toString() {
return dump.dumpToString(toMap());
}
}

43
nb-api/src/main/java/io/nosqlbench/nb/api/components/status/HeartbeatRepresenter.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.status;
import io.nosqlbench.nb.api.components.core.NBInvokableState;
import org.snakeyaml.engine.v2.api.DumpSettings;
import org.snakeyaml.engine.v2.api.RepresentToNode;
import org.snakeyaml.engine.v2.nodes.Node;
import org.snakeyaml.engine.v2.representer.StandardRepresenter;
public class HeartbeatRepresenter extends StandardRepresenter {
public HeartbeatRepresenter(DumpSettings settings) {
super(settings);
this.representers.put(NBInvokableState.class, new RepresentEnumToString());
}
public class RepresentEnumToString implements RepresentToNode {
@Override
public Node representData(Object o) {
if (o instanceof Enum<?> e) {
String name = e.name();
return HeartbeatRepresenter.this.represent(name);
} else {
throw new RuntimeException("Unable to represent as enum: " + o.toString() + " (class " + o.getClass().getSimpleName() + "'");
}
}
}
}

63
nb-api/src/main/java/io/nosqlbench/nb/api/components/status/NBLiveComponent.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.api.components.status;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBInvokableState;
import io.nosqlbench.nb.api.labels.NBLabels;
import java.util.Map;
/**
* A <EM>live</EM> component is one which provides evidence that it is either
* in a healthy state or that it is not, via a heartbeat mechanism.
*/
public class NBLiveComponent extends NBBaseComponent {
public NBLiveComponent(NBComponent parentComponent) {
super(parentComponent);
}
public NBLiveComponent(NBComponent parentComponent, NBLabels componentSpecificLabelsOnly, Map<String, String> props, String liveLabel) {
super(parentComponent, componentSpecificLabelsOnly, props);
// attaches, no further reference needed
new ComponentPulse(this, NBLabels.forKV(), liveLabel, Long.parseLong(getComponentProp("heartbeat").orElse("60000")));
}
public Heartbeat heartbeat() {
return new Heartbeat(
getLabels(),
this.getComponentState(),
started_ns,
sessionTimeMs(),
0L,
0L
);
}
private long sessionTimeMs() {
NBInvokableState state = getComponentState();
long nanos = switch (state) {
case ERRORED -> (nanosof_error() - nanosof_start());
case STARTING, RUNNING -> (System.nanoTime() - nanosof_start());
case CLOSING -> (nanosof_close() - nanosof_start());
case STOPPED -> (nanosof_teardown() - nanosof_start());
};
return nanos / 1_000_000L;
}
}

2
nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/ConsoleReporter.java
View File

@ -44,7 +44,7 @@ public class ConsoleReporter extends PeriodicTaskComponent {
public ConsoleReporter(NBComponent node, NBLabels extraLabels, long millis, boolean oneLastTime,
PrintStream output, Set<MetricAttribute> disabledMetricAttributes) {
super(node, extraLabels, millis, oneLastTime, "REPORT-CONSOLE");
super(node, extraLabels, millis, "REPORT-CONSOLE", FirstReport.OnInterval, LastReport.OnInterrupt);
this.output = output;
this.dateFormat = DateFormat.getDateTimeInstance(DateFormat.SHORT,
DateFormat.MEDIUM,

2
nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/CsvReporter.java
View File

@ -59,7 +59,7 @@ public class CsvReporter extends PeriodicTaskComponent {
public CsvReporter(NBComponent node, Path reportTo, long intervalMs, MetricInstanceFilter filter,
NBLabels extraLabels) {
super(node, extraLabels, intervalMs, false,"REPORT-CSV");
super(node, extraLabels, intervalMs, "REPORT-CSV", FirstReport.OnInterval, LastReport.OnInterrupt);
this.component = node;
this.reportTo = reportTo;
this.filter = filter;

2
nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/Log4JMetricsReporter.java
View File

@ -51,7 +51,7 @@ public class Log4JMetricsReporter extends PeriodicTaskComponent {
final NBLabels extraLabels,
final long millis,
final boolean oneLastTime) {
super(component, extraLabels, millis, oneLastTime,"REPORT-LOG4J");
super(component, extraLabels, millis, "REPORT-LOG4J", FirstReport.OnInterval, LastReport.OnInterrupt);
this.loggerProxy = loggerProxy;
this.marker = marker;
}

2
nb-api/src/main/java/io/nosqlbench/nb/api/engine/metrics/reporters/PromPushReporterComponent.java
View File

@ -51,7 +51,7 @@ public class PromPushReporterComponent extends PeriodicTaskComponent {
private String bearerToken;
public PromPushReporterComponent(NBComponent parent, String endpoint, long intervalMs, NBLabels nbLabels) {
super(parent, nbLabels.and("_type", "prom-push"), intervalMs, true, "REPORT-PROMPUSH");
super(parent, nbLabels.and("_type", "prom-push"), intervalMs, "REPORT-PROMPUSH",FirstReport.OnInterval, LastReport.OnInterrupt);
String jobname = getLabels().valueOfOptional("jobname").orElse("default");
String instance = getLabels().valueOfOptional("instance").orElse("default");
if (jobname.equals("default") || instance.equals("default")) {

3
nbr-examples/src/test/java/io/nosqlbench/nbr/examples/injavascript/ScenarioExampleTests.java
View File

@ -30,6 +30,7 @@ import org.junit.jupiter.api.parallel.Execution;
import org.junit.jupiter.api.parallel.ExecutionMode;
import java.util.List;
import java.util.Map;
@Disabled
@Execution(ExecutionMode.SAME_THREAD)
@ -44,7 +45,7 @@ public class ScenarioExampleTests {
NBCLIOptions parser = new NBCLIOptions(params);
List<Cmd> commands = parser.getCommands();
var myroot = new TestComponent("test_"+params[0]);
NBSession session = new NBSession(myroot,"session_"+params[0]);
NBSession session = new NBSession(myroot,"session_"+params[0], Map.of());
System.out.println("=".repeat(29) + " Running scenario test for example scenario: " + params[0]);
ExecutionResult result = session.apply(commands);
return result;