IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

provide better defaults for summary reporting

Browse Source
This commit is contained in:
Jonathan Shook 2021-01-13 14:15:19 -06:00
parent 1184fd77e9
commit 69747056ab
4 changed files with 101 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -342,10 +342,13 @@ public class NBCLI {
options.wantsStackTraces(),
options.wantsCompileScript(),
options.getReportSummaryTo(),
String.join("\n", args)
String.join("\n", args),
options.getLogsDirectory()
);
ScriptBuffer buffer = new BasicScriptBuffer()
.add(options.getCommands().toArray(new Cmd[0]));
.add(options.getCommands()
.toArray(new Cmd[0]));
String scriptData = buffer.getParsedScript();
if (options.wantsShowScript()) {
@ -387,7 +390,7 @@ public class NBCLI {
ScenariosResults scenariosResults = executor.awaitAllResults();
ActivityMetrics.closeMetrics(options.wantsEnableChart());
scenariosResults.reportToLog();
//scenariosResults.reportToLog();
ShutdownManager.shutdown();
// logger.info(scenariosResults.getExecutionSummary());

8
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -77,6 +77,7 @@ public class NBCLIOptions {
private static final String REPORT_GRAPHITE_TO = "--report-graphite-to";
private static final String REPORT_CSV_TO = "--report-csv-to";
private static final String REPORT_SUMMARY_TO = "--report-summary-to";
private final static String REPORT_SUMMARY_TO_DEFAULT = "stdout:60,_LOGS_/_SESSION_.summary";
private static final String PROGRESS = "--progress";
private static final String WITH_LOGGING_PATTERN = "--with-logging-pattern";
private static final String LOG_HISTOGRAMS = "--log-histograms";
@ -84,6 +85,7 @@ public class NBCLIOptions {
private static final String CLASSIC_HISTOGRAMS = "--classic-histograms";
private final static String LOG_LEVEL_OVERRIDE = "--log-level-override";
private final static String ENABLE_CHART = "--enable-chart";
private final static String DOCKER_METRICS = "--docker-metrics";
private final static String DOCKER_METRICS_AT = "--docker-metrics-at";
private static final String DOCKER_GRAFANA_TAG = "--docker-grafana-tag";
@ -150,7 +152,7 @@ public class NBCLIOptions {
private final List<String> statePathAccesses = new ArrayList<>();
private final String hdrForChartFileName = DEFAULT_CHART_HDR_LOG_NAME;
private String dockerPromRetentionDays = "183d";
private String reportSummaryTo = "stdout:60";
private String reportSummaryTo = REPORT_SUMMARY_TO_DEFAULT;
public String getAnnotatorsConfig() {
return annotatorsConfig;
@ -498,7 +500,7 @@ public class NBCLIOptions {
break;
case REPORT_SUMMARY_TO:
arglist.removeFirst();
reportSummaryTo = arglist.removeFirst();
reportSummaryTo = readWordOrThrow(arglist, "report summary file");
break;
case LIST_DRIVERS:
case LIST_ACTIVITY_TYPES:
@ -726,7 +728,7 @@ public class NBCLIOptions {
public String getScriptFile() {
if (scriptFile == null) {
return logsDirectory + File.separator + "_SESSIONNAME_" + ".js";
return logsDirectory + File.separator + "_SESSION_" + ".js";
}
String expanded = scriptFile;

36
engine-cli/src/main/resources/commandline.md
View File

@ -163,6 +163,7 @@ with the default credentials admin/admin).
### Console Options ###
Increase console logging levels: (Default console logging level is *warning*)
-v (info)
@ -172,8 +173,41 @@ Increase console logging levels: (Default console logging level is *warning*)
--progress console:1m (disables itself if -v options are used)
These levels affect *only* the console output level. Other logging level
parameters affect logging to the scenario log, stored by default in logs/...
parameters affect logging to the scenario log, stored by default in
logs/...
Show version, long form, with artifact coordinates.
--version
### Summary Reporting
The classic metrics logging format is used to report results into the
logfile for every scenario. This format is not generally human-friendly,
so a better summary report is provided by default to the console and/or a
specified summary file by default.
Examples:
# report to console if session ran more than 60 seconds
--report-summary-to stdout:60
# report to auto-named summary file for every session
--report-summary-to _LOGS_/_SESSION_.summary
# do both (the default)
--report-summary-to stdout:60,_LOGS_/_SESSION_.summary
Values of `stdout` or `stderr` are send summaries directly to the console,
and any other pattern is taken as a file name.
You can use `_SESSION_` and `_LOGS_` to automatically name the file
according to the current session name and log directory.
The reason for the optional timing parameter is to allow for results of
short scenario runs to be squelched. Metrics for short runs are not
generally accurate nor meaningful. Spamming the console with boiler-plate
in such cases is undesirable. If the minimum session length is not
specified, it is assumed to be 0, meaning that a report will always show
on that channel.

97
engine-core/src/main/java/io/nosqlbench/engine/core/script/Scenario.java
View File

@ -57,6 +57,7 @@ public class Scenario implements Callable<ScenarioResult> {
private final String commandLine;
private final String reportSummaryTo;
private final Path logsPath;
private Logger logger = LogManager.getLogger("SCENARIO");
private State state = State.Scheduled;
@ -103,7 +104,9 @@ public class Scenario implements Callable<ScenarioResult> {
boolean wantsStackTraces,
boolean wantsCompiledScript,
String reportSummaryTo,
String commandLine) {
String commandLine,
Path logsPath) {
this.scenarioName = scenarioName;
this.scriptfile = scriptfile;
this.engine = engine;
@ -113,6 +116,15 @@ public class Scenario implements Callable<ScenarioResult> {
this.wantsCompiledScript = wantsCompiledScript;
this.reportSummaryTo = reportSummaryTo;
this.commandLine = commandLine;
this.logsPath = logsPath;
}
public Scenario(String name, Engine engine) {
this.scenarioName = name;
this.reportSummaryTo = "CONSOLE";
this.engine = engine;
this.commandLine = "";
this.logsPath = Path.of("logs");
}
public Scenario setLogger(Logger logger) {
@ -124,13 +136,6 @@ public class Scenario implements Callable<ScenarioResult> {
return logger;
}
public Scenario(String name, Engine engine) {
this.scenarioName = name;
this.reportSummaryTo = "CONSOLE";
this.engine = engine;
this.commandLine = "";
}
public Scenario addScriptText(String scriptText) {
scripts.add(scriptText);
return this;
@ -156,7 +161,7 @@ public class Scenario implements Callable<ScenarioResult> {
private void init() {
logger.info("Using engine " + engine.toString());
logger.debug("Using engine " + engine.toString());
MetricRegistry metricRegistry = ActivityMetrics.getMetricRegistry();
@ -239,7 +244,7 @@ public class Scenario implements Callable<ScenarioResult> {
}
public void run() {
public void runScenario() {
scenarioShutdownHook = new ScenarioShutdownHook(this);
Runtime.getRuntime().addShutdownHook(scenarioShutdownHook);
@ -269,7 +274,7 @@ public class Scenario implements Callable<ScenarioResult> {
} else {
if (scriptfile != null && !scriptfile.isEmpty()) {
String filename = scriptfile.replace("_SESSIONNAME_", scenarioName);
String filename = scriptfile.replace("_SESSION_", scenarioName);
logger.debug("-> invoking main scenario script (" +
"interpreted from " + filename + ")");
Path written = Files.write(
@ -354,49 +359,57 @@ public class Scenario implements Callable<ScenarioResult> {
}
public ScenarioResult call() {
run();
runScenario();
String iolog = scriptEnv.getTimedLog();
ScenarioResult result = new ScenarioResult(iolog, this.startedAtMillis, this.endedAtMillis);
result.reportToLog();
Optional.ofNullable(getSummaryDestination(reportSummaryTo, result.getElapsedMillis()))
.ifPresent(result::reportTo);
getSummaryDestinations(reportSummaryTo, result.getElapsedMillis()).forEach(result::reportTo);
return result;
}
private PrintStream getSummaryDestination(String reportSummaryTo, long elapsedMillis) {
if (reportSummaryTo != null && !reportSummaryTo.isBlank()) {
String[] split = reportSummaryTo.split(":", 2);
String summaryTo = split[0];
long summaryWhen = split.length == 2 ? Long.parseLong(split[1]) * 1000L : 60000L;
if (elapsedMillis > summaryWhen) {
PrintStream out = null;
switch (summaryTo.toLowerCase()) {
case "console":
case "stdout":
return System.out;
case "stderr":
return System.err;
default:
String outName = summaryTo
.replaceAll("_session_", getScenarioName())
.replaceAll("_scenario_", getScenarioName());
try {
out = new PrintStream(new FileOutputStream(outName));
return out;
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
private List<PrintStream> getSummaryDestinations(String reportSummaryTo, long elapsedMillis) {
List<PrintStream> destinations = new ArrayList<>();
String[] destinationSpecs = reportSummaryTo.split(", *");
for (String spec : destinationSpecs) {
if (spec != null && !spec.isBlank()) {
String[] split = spec.split(":", 2);
String summaryTo = split[0];
long summaryWhen = split.length == 2 ? Long.parseLong(split[1]) * 1000L : 0;
if (elapsedMillis > summaryWhen) {
PrintStream out = null;
switch (summaryTo.toLowerCase()) {
case "console":
case "stdout":
destinations.add(System.out);
break;
case "stderr":
destinations.add(System.err);
break;
default:
String outName = summaryTo
.replaceAll("_SESSION_", getScenarioName())
.replaceAll("_LOGS_", logsPath.toString());
try {
out = new PrintStream(new FileOutputStream(outName));
destinations.add(out);
break;
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
} else {
logger.debug("Suppressing metrics report to " + spec + " with scenario duration of " + summaryWhen + "ms");
}
} else {
logger.info("Metrics suppressed because scenario was less than " + summaryWhen + "ms long.");
logger.info("Metrics data is not reliable for short sampling periods.");
logger.info("To get metrics on console, run a longer scenario.");
}
}
return null;
return destinations;
}
@Override