provide better defaults for summary reporting

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -342,10 +342,13 @@ public class NBCLI {
             options.wantsStackTraces(),
             options.wantsCompileScript(),
             options.getReportSummaryTo(),
-            String.join("\n", args)
+            String.join("\n", args),
+            options.getLogsDirectory()
         );
+
         ScriptBuffer buffer = new BasicScriptBuffer()
-                .add(options.getCommands().toArray(new Cmd[0]));
+            .add(options.getCommands()
+                .toArray(new Cmd[0]));
         String scriptData = buffer.getParsedScript();
 
         if (options.wantsShowScript()) {
@@ -387,7 +390,7 @@ public class NBCLI {
         ScenariosResults scenariosResults = executor.awaitAllResults();
 
         ActivityMetrics.closeMetrics(options.wantsEnableChart());
-        scenariosResults.reportToLog();
+        //scenariosResults.reportToLog();
         ShutdownManager.shutdown();
 
 //        logger.info(scenariosResults.getExecutionSummary());

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -77,6 +77,7 @@ public class NBCLIOptions {
     private static final String REPORT_GRAPHITE_TO = "--report-graphite-to";
     private static final String REPORT_CSV_TO = "--report-csv-to";
     private static final String REPORT_SUMMARY_TO = "--report-summary-to";
+    private final static String REPORT_SUMMARY_TO_DEFAULT = "stdout:60,_LOGS_/_SESSION_.summary";
     private static final String PROGRESS = "--progress";
     private static final String WITH_LOGGING_PATTERN = "--with-logging-pattern";
     private static final String LOG_HISTOGRAMS = "--log-histograms";
@@ -84,6 +85,7 @@ public class NBCLIOptions {
     private static final String CLASSIC_HISTOGRAMS = "--classic-histograms";
     private final static String LOG_LEVEL_OVERRIDE = "--log-level-override";
     private final static String ENABLE_CHART = "--enable-chart";
+
     private final static String DOCKER_METRICS = "--docker-metrics";
     private final static String DOCKER_METRICS_AT = "--docker-metrics-at";
     private static final String DOCKER_GRAFANA_TAG = "--docker-grafana-tag";
@@ -150,7 +152,7 @@ public class NBCLIOptions {
     private final List<String> statePathAccesses = new ArrayList<>();
     private final String hdrForChartFileName = DEFAULT_CHART_HDR_LOG_NAME;
     private String dockerPromRetentionDays = "183d";
-    private String reportSummaryTo = "stdout:60";
+    private String reportSummaryTo = REPORT_SUMMARY_TO_DEFAULT;
 
     public String getAnnotatorsConfig() {
         return annotatorsConfig;
@@ -498,7 +500,7 @@ public class NBCLIOptions {
                     break;
                 case REPORT_SUMMARY_TO:
                     arglist.removeFirst();
-                    reportSummaryTo = arglist.removeFirst();
+                    reportSummaryTo = readWordOrThrow(arglist, "report summary file");
                     break;
                 case LIST_DRIVERS:
                 case LIST_ACTIVITY_TYPES:
@@ -726,7 +728,7 @@ public class NBCLIOptions {
 
     public String getScriptFile() {
         if (scriptFile == null) {
-            return logsDirectory + File.separator + "_SESSIONNAME_" + ".js";
+            return logsDirectory + File.separator + "_SESSION_" + ".js";
         }
 
         String expanded = scriptFile;

engine-cli/src/main/resources/commandline.md

@@ -163,6 +163,7 @@ with the default credentials admin/admin).
 
 
 ### Console Options ###
+
 Increase console logging levels: (Default console logging level is *warning*)
 
     -v         (info)
@@ -172,8 +173,41 @@ Increase console logging levels: (Default console logging level is *warning*)
     --progress console:1m (disables itself if -v options are used)
 
 These levels affect *only* the console output level. Other logging level
-parameters affect logging to the scenario log, stored by default in logs/...
+parameters affect logging to the scenario log, stored by default in
+logs/...
 
 Show version, long form, with artifact coordinates.
 
     --version
+
+### Summary Reporting
+
+The classic metrics logging format is used to report results into the
+logfile for every scenario. This format is not generally human-friendly,
+so a better summary report is provided by default to the console and/or a
+specified summary file by default.
+
+Examples:
+
+    # report to console if session ran more than 60 seconds
+    --report-summary-to stdout:60
+
+    # report to auto-named summary file for every session
+    --report-summary-to _LOGS_/_SESSION_.summary
+
+    # do both (the default)
+    --report-summary-to stdout:60,_LOGS_/_SESSION_.summary
+
+Values of `stdout` or `stderr` are send summaries directly to the console,
+and any other pattern is taken as a file name.
+
+You can use `_SESSION_` and `_LOGS_` to automatically name the file
+according to the current session name and log directory.
+
+The reason for the optional timing parameter is to allow for results of
+short scenario runs to be squelched. Metrics for short runs are not
+generally accurate nor meaningful. Spamming the console with boiler-plate
+in such cases is undesirable. If the minimum session length is not
+specified, it is assumed to be 0, meaning that a report will always show
+on that channel.
+

engine-core/src/main/java/io/nosqlbench/engine/core/script/Scenario.java

@@ -57,6 +57,7 @@ public class Scenario implements Callable<ScenarioResult> {
 
     private final String commandLine;
     private final String reportSummaryTo;
+    private final Path logsPath;
     private Logger logger = LogManager.getLogger("SCENARIO");
 
     private State state = State.Scheduled;
@@ -103,7 +104,9 @@ public class Scenario implements Callable<ScenarioResult> {
         boolean wantsStackTraces,
         boolean wantsCompiledScript,
         String reportSummaryTo,
-        String commandLine) {
+        String commandLine,
+        Path logsPath) {
+
         this.scenarioName = scenarioName;
         this.scriptfile = scriptfile;
         this.engine = engine;
@@ -113,6 +116,15 @@ public class Scenario implements Callable<ScenarioResult> {
         this.wantsCompiledScript = wantsCompiledScript;
         this.reportSummaryTo = reportSummaryTo;
         this.commandLine = commandLine;
+        this.logsPath = logsPath;
+    }
+
+    public Scenario(String name, Engine engine) {
+        this.scenarioName = name;
+        this.reportSummaryTo = "CONSOLE";
+        this.engine = engine;
+        this.commandLine = "";
+        this.logsPath = Path.of("logs");
     }
 
     public Scenario setLogger(Logger logger) {
@@ -124,13 +136,6 @@ public class Scenario implements Callable<ScenarioResult> {
         return logger;
     }
 
-    public Scenario(String name, Engine engine) {
-        this.scenarioName = name;
-        this.reportSummaryTo = "CONSOLE";
-        this.engine = engine;
-        this.commandLine = "";
-    }
-
     public Scenario addScriptText(String scriptText) {
         scripts.add(scriptText);
         return this;
@@ -156,7 +161,7 @@ public class Scenario implements Callable<ScenarioResult> {
 
     private void init() {
 
-        logger.info("Using engine " + engine.toString());
+        logger.debug("Using engine " + engine.toString());
 
         MetricRegistry metricRegistry = ActivityMetrics.getMetricRegistry();
 
@@ -239,7 +244,7 @@ public class Scenario implements Callable<ScenarioResult> {
 
     }
 
-    public void run() {
+    public void runScenario() {
         scenarioShutdownHook = new ScenarioShutdownHook(this);
         Runtime.getRuntime().addShutdownHook(scenarioShutdownHook);
 
@@ -269,7 +274,7 @@ public class Scenario implements Callable<ScenarioResult> {
                 } else {
                     if (scriptfile != null && !scriptfile.isEmpty()) {
 
-                        String filename = scriptfile.replace("_SESSIONNAME_", scenarioName);
+                        String filename = scriptfile.replace("_SESSION_", scenarioName);
                         logger.debug("-> invoking main scenario script (" +
                             "interpreted from " + filename + ")");
                         Path written = Files.write(
@@ -354,49 +359,57 @@ public class Scenario implements Callable<ScenarioResult> {
     }
 
     public ScenarioResult call() {
-        run();
+        runScenario();
         String iolog = scriptEnv.getTimedLog();
         ScenarioResult result = new ScenarioResult(iolog, this.startedAtMillis, this.endedAtMillis);
 
         result.reportToLog();
 
-        Optional.ofNullable(getSummaryDestination(reportSummaryTo, result.getElapsedMillis()))
+        getSummaryDestinations(reportSummaryTo, result.getElapsedMillis()).forEach(result::reportTo);
-            .ifPresent(result::reportTo);
 
         return result;
     }
 
-    private PrintStream getSummaryDestination(String reportSummaryTo, long elapsedMillis) {
+    private List<PrintStream> getSummaryDestinations(String reportSummaryTo, long elapsedMillis) {
-        if (reportSummaryTo != null && !reportSummaryTo.isBlank()) {
+        List<PrintStream> destinations = new ArrayList<>();
-            String[] split = reportSummaryTo.split(":", 2);
+
-            String summaryTo = split[0];
+
-            long summaryWhen = split.length == 2 ? Long.parseLong(split[1]) * 1000L : 60000L;
+        String[] destinationSpecs = reportSummaryTo.split(", *");
-            if (elapsedMillis > summaryWhen) {
+
-                PrintStream out = null;
+        for (String spec : destinationSpecs) {
-                switch (summaryTo.toLowerCase()) {
+            if (spec != null && !spec.isBlank()) {
-                    case "console":
+                String[] split = spec.split(":", 2);
-                    case "stdout":
+                String summaryTo = split[0];
-                        return System.out;
+                long summaryWhen = split.length == 2 ? Long.parseLong(split[1]) * 1000L : 0;
-                    case "stderr":
+                if (elapsedMillis > summaryWhen) {
-                        return System.err;
+                    PrintStream out = null;
-                    default:
+                    switch (summaryTo.toLowerCase()) {
-                        String outName = summaryTo
+                        case "console":
-                            .replaceAll("_session_", getScenarioName())
+                        case "stdout":
-                            .replaceAll("_scenario_", getScenarioName());
+                            destinations.add(System.out);
-                        try {
+                            break;
-                            out = new PrintStream(new FileOutputStream(outName));
+                        case "stderr":
-                            return out;
+                            destinations.add(System.err);
-                        } catch (FileNotFoundException e) {
+                            break;
-                            throw new RuntimeException(e);
+                        default:
-                        }
+                            String outName = summaryTo
+                                .replaceAll("_SESSION_", getScenarioName())
+                                .replaceAll("_LOGS_", logsPath.toString());
+                            try {
+                                out = new PrintStream(new FileOutputStream(outName));
+                                destinations.add(out);
+                                break;
+                            } catch (FileNotFoundException e) {
+                                throw new RuntimeException(e);
+                            }
+                    }
+                } else {
+                    logger.debug("Suppressing metrics report to " + spec + " with scenario duration of " + summaryWhen + "ms");
                 }
-            } else {
-                logger.info("Metrics suppressed because scenario was less than " + summaryWhen + "ms long.");
-                logger.info("Metrics data is not reliable for short sampling periods.");
-                logger.info("To get metrics on console, run a longer scenario.");
             }
         }
-        return null;
+
+        return destinations;
     }
 
     @Override