argsfiles and statedirs

engine-cli/src/main/java/io/nosqlbench/engine/cli/ArgsFile.java

@@ -1,234 +0,0 @@
-package io.nosqlbench.engine.cli;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-/**
- * <H1>Synopsis</H1>
- *
- * ArgsFile is a command-line modifier which can take linked list of
- * command args and modify it, and/or modify argsfile refrenced in this way.
- *
- * <H1>ArgsFile Selection</H1>
- *
- * During processing, any occurence of '--argsfile' selects the active argsfile and loads
- * it into the command line in place of the '--argsfile' argument. By default the args file
- * will be loaded if it exists, and a warning will be given if it does not.
- *
- * The '--argsfile-required &lt;somepath&gt;' version will throw an error if the args file
- * is not present, but it will not report any warnings or details otherwise.
- *
- * The `--argsfile-optional &lt;somepath&gt; version will not throw an error if the args
- * file is not present, and it will not report any warnings or details otherwise.
- *
- * A prefix command line can be given to ArgsFile to pre-load any settings. In this way
- * it is possible to easily provide a default args file which will be loaded. For example,
- * A prefix command of '--argsfile-optional &lt;somepath&gt;' will load options if they are
- * available in the specified file, but will otherwise provide no feedback to the user.
- *
- * <H1>ArgsFile Injection</H1>
- *
- * When an argsfile is loaded, it reads a command from each line into the current position
- * of the command line. No parsing is done. Blank lines are ignored. Newlines are used as the
- * argument delimiter, and lines that end with a backslash before the newline are automatically
- * joined together.
- *
- * <H1>ArgsFile Diagnostics</H1>
- *
- * All modifications to the command line should be reported to the logging facility at
- * INFO level. This assumes that the calling layer wants to inform users of command line injections,
- * and that the user can select to be notified of warnings only if desired.
- *
- * <H1>Environment Variables</H1>
- *
- * Simple environment variable substitution is attempted for any pattern which appears as '$' followed
- * by all uppercase letters and underscores. Any references of this type which are not resolvable
- * will cause an error to be thrown.
- */
-public class ArgsFile {
-    private final static Logger logger = LoggerFactory.getLogger(ArgsFile.class);
-
-    private Path argsPath;
-    private LinkedList<String> preload;
-
-    public ArgsFile() {
-    }
-
-    public ArgsFile preload(String... preload) {
-        this.preload = new LinkedList<String>(Arrays.asList(preload));
-        return this;
-    }
-
-    private enum Selection {
-        // Ignore if not present, show injections at info
-        IgnoreIfMissing,
-        // Warn if not present, but show injections at info
-        WarnIfMissing,
-        // throw error if not present, show injections at info
-        ErrorIfMissing
-    }
-
-    public LinkedList<String> process(String... args) {
-        return process(new LinkedList<String>(Arrays.asList(args)));
-    }
-
-    public LinkedList<String> process(LinkedList<String> commandline) {
-        if (preload != null) {
-            LinkedList<String> modified = new LinkedList<String>();
-            modified.addAll(preload);
-            modified.addAll(commandline);
-            preload = null;
-            commandline = modified;
-        }
-        LinkedList<String> composed = new LinkedList<>();
-        while (commandline.peekFirst() != null) {
-            String arg = commandline.peekFirst();
-            switch (arg) {
-                case "--argsfile":
-                    commandline.removeFirst();
-                    String argspath = readWordOrThrow(commandline, "path to an args file");
-                    setArgsFile(argspath, Selection.WarnIfMissing);
-                    commandline = loadArgs(this.argsPath, Selection.WarnIfMissing, commandline);
-                    break;
-                case "--argsfile-required":
-                    commandline.removeFirst();
-                    String argspathRequired = readWordOrThrow(commandline, "path to an args file");
-                    setArgsFile(argspathRequired, Selection.ErrorIfMissing);
-                    commandline = loadArgs(this.argsPath, Selection.ErrorIfMissing, commandline);
-                    break;
-                case "--argsfile-optional":
-                    commandline.removeFirst();
-                    String argspathOptional = readWordOrThrow(commandline, "path to an args file");
-                    setArgsFile(argspathOptional, Selection.IgnoreIfMissing);
-                    commandline = loadArgs(this.argsPath, Selection.IgnoreIfMissing, commandline);
-                    break;
-                case "--pin":
-                    commandline.removeFirst();
-                    commandline = pinArg(commandline);
-                    break;
-                case "--unpin":
-                    commandline.removeFirst();
-                    commandline = unpinArg(commandline);
-                    break;
-                default:
-                    composed.addLast(commandline.removeFirst());
-            }
-
-        }
-        return composed;
-    }
-
-    private LinkedList<String> loadArgs(Path argspath, Selection mode, LinkedList<String> commandline) {
-        if (!assertArgsFileExists(argspath, mode)) {
-            return commandline;
-        }
-        List<String> lines = null;
-        try {
-            lines = Files.readAllLines(argspath);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        List<String> content = lines.stream()
-                .filter(s -> !s.startsWith("#"))
-                .filter(s -> !s.startsWith("/"))
-                .filter(s -> !s.isBlank())
-                .filter(s -> !s.isEmpty())
-                .collect(Collectors.toList());
-        StringBuilder splitword = new StringBuilder();
-        LinkedList<String> loaded = new LinkedList<>();
-        for (String s : content) {
-            splitword.append(s);
-            if (!s.endsWith("\\")) {
-                loaded.addLast(splitword.toString());
-                splitword.setLength(0);
-            } else {
-                splitword.setLength(splitword.length() - 1);
-            }
-        }
-        if (splitword.length() > 0) {
-            throw new RuntimeException("unqualified line continuation for '" + splitword.toString() + "'");
-        }
-
-        Iterator<String> injections = loaded.descendingIterator();
-        while (injections.hasNext()) {
-            String injection = injections.next();
-            injection = injectEnv(injection);
-            commandline.addFirst(injection);
-        }
-
-        return commandline;
-    }
-
-    private boolean assertArgsFileExists(Path argspath, Selection mode) {
-        if (!Files.exists(argsPath)) {
-            switch (mode) {
-                case ErrorIfMissing:
-                    throw new RuntimeException("A required argsfile was specified, but it does not exist: '" + argspath + "'");
-                case WarnIfMissing:
-                    logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'");
-                case IgnoreIfMissing:
-            }
-            return false;
-        }
-        return true;
-    }
-
-    private void setArgsFile(String argspath, Selection mode) {
-        this.argsPath = Path.of(argspath);
-//        assertIfMissing(this.argsPath,mode);
-    }
-
-    private String readWordOrThrow(LinkedList<String> commandline, String description) {
-        String found = commandline.peekFirst();
-        if (found == null) {
-            throw new RuntimeException("Unable to read argument top option for " + description);
-        }
-        return commandline.removeFirst();
-    }
-
-    private LinkedList<String> pinArg(LinkedList<String> commandline) {
-        if (this.argsPath == null) {
-            throw new RuntimeException("No argsfile has been selected before using the pin option.");
-        }
-        return commandline;
-    }
-
-    private LinkedList<String> unpinArg(LinkedList<String> commandline) {
-        if (this.argsPath == null) {
-            throw new RuntimeException("No argsfile has been selected before using the unpin option.");
-        }
-        return commandline;
-    }
-
-    private String injectEnv(String word) {
-        Pattern envpattern = Pattern.compile("(?<envvar>\\$[A-Z_]+)");
-        Matcher matcher = envpattern.matcher(word);
-        StringBuilder sb = new StringBuilder();
-        while (matcher.find()) {
-            String envvar = matcher.group("envvar");
-            String value = System.getenv(envvar);
-            if (value == null) {
-                throw new RuntimeException("Env var '" + envvar + "' was not found in the environment.");
-            }
-            matcher.appendReplacement(sb, value);
-        }
-        matcher.appendTail(sb);
-        return sb.toString();
-    }
-
-    public LinkedList<String> pin(LinkedList<String> arglist) {
-        return arglist;
-    }
-
-    public LinkedList<String> unpin(LinkedList<String> arglist) {
-        return arglist;
-    }
-}

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -39,10 +39,10 @@ import java.util.stream.Collectors;
 
 public class NBCLI {
 
-    private static final Logger logger = LoggerFactory.getLogger(NBCLI.class);
+    private static final Logger logger = LoggerFactory.getLogger("NBCLI");
     private static final Logger EVENTS = LoggerFactory.getLogger("EVENTS");
-    private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
 
+    private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
 
     private final String commandName;
 
@@ -70,36 +70,43 @@ public class NBCLI {
         NBCLIOptions globalOptions = new NBCLIOptions(args, NBCLIOptions.Mode.ParseGlobalsOnly);
 
         // Global only processing
+        if (args.length == 0) {
+            System.out.println(loadHelpFile("commandline.md"));
+            System.exit(0);
+        }
 
+        boolean dockerMetrics = globalOptions.wantsDockerMetrics();
+        String dockerMetricsAt = globalOptions.wantsDockerMetricsAt();
         String reportGraphiteTo = globalOptions.wantsReportGraphiteTo();
+        int mOpts = (dockerMetrics ? 1 : 0) + (dockerMetricsAt != null ? 1 : 0) + (reportGraphiteTo != null ? 1 : 0);
+        if (mOpts > 1) {
+            throw new BasicError("You have multiple conflicting options which attempt to set\n" +
+                    " the destination for metrics and annotations. Please select only one of\n" +
+                    " --docker-metrics, --docker-metrics-at <addr>, or --report-graphite-to <addr>\n" +
+                    " For more details, see run 'nb help docker-metrics'");
+        }
 
-        if (globalOptions.wantsDockerMetrics()) {
+        String metricsAddr = null;
+
+        if (dockerMetrics) {
+            // Setup docker stack for local docker metrics
             logger.info("Docker metrics is enabled. Docker must be installed for this to work");
             DockerMetricsManager dmh = new DockerMetricsManager();
             Map<String, String> dashboardOptions = Map.of(
                     DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag()
             );
             dmh.startMetrics(dashboardOptions);
-
             String warn = "Docker Containers are started, for grafana and prometheus, hit" +
                     " these urls in your browser: http://<host>:3000 and http://<host>:9090";
             logger.warn(warn);
-            if (reportGraphiteTo != null) {
+            metricsAddr = "localhost";
-                logger.warn(String.format("Docker metrics are enabled (--docker-metrics)" +
+        } else if (dockerMetricsAt != null) {
-                                " but graphite reporting (--report-graphite-to) is set to %s \n" +
+            metricsAddr = dockerMetricsAt;
-                                "usually only one of the two is configured.",
-                        reportGraphiteTo));
-            } else {
-                logger.info("Setting graphite reporting to localhost");
-                reportGraphiteTo = "localhost:9109";
         }
 
-            if (globalOptions.getAnnotatorsConfig() != null) {
+        if (metricsAddr != null) {
-                logger.warn("Docker metrics and separate annotations" +
+            reportGraphiteTo = metricsAddr + ":9109";
-                        "are configured (both --docker-metrics and --annotations).");
+            Annotators.init("{type:'grafana',url:'http://" + metricsAddr + ":3000/'}");
-            } else {
-                Annotators.init("grafana{http://localhost:3000/}");
-            }
         }
 
         if (args.length > 0 && args[0].toLowerCase().equals("virtdata")) {
@@ -275,13 +282,6 @@ public class NBCLI {
         // intentionally not shown for warn-only
         logger.info("console logging level is " + options.wantsConsoleLogLevel());
 
-        if (options.getCommands().
-
-                size() == 0) {
-            System.out.println(loadHelpFile("commandline.md"));
-            System.exit(0);
-        }
-
         ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1);
 
         Scenario scenario = new Scenario(
@@ -311,8 +311,6 @@ public class NBCLI {
         }
 
 
-        // Execute Scenario!
-
         Level consoleLogLevel = options.wantsConsoleLogLevel();
         Level scenarioLogLevel = Level.toLevel(options.getLogsLevel());
         if (scenarioLogLevel.toInt() > consoleLogLevel.toInt()) {
@@ -321,6 +319,12 @@ public class NBCLI {
 
         Level maxLevel = Level.toLevel(Math.min(consoleLogLevel.toInt(), scenarioLogLevel.toInt()));
 
+        // Execute Scenario!
+        if (options.getCommands().size() == 0) {
+            logger.info("No commands provided. Exiting before scenario.");
+            System.exit(0);
+        }
+
         scenario.addScriptText(scriptData);
         ScriptParams scriptParams = new ScriptParams();
         scriptParams.putAll(buffer.getCombinedParams());

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIArgsFile.java

@@ -0,0 +1,500 @@
+package io.nosqlbench.engine.cli;
+
+import io.nosqlbench.nb.api.errors.BasicError;
+import joptsimple.internal.Strings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * <H1>Synopsis</H1>
+ *
+ * ArgsFile is a command-line modifier which can take linked list of
+ * command args and modify it, and/or modify argsfile refrenced in this way.
+ *
+ * <H1>ArgsFile Selection</H1>
+ *
+ * During processing, any occurence of '--argsfile' selects the active argsfile and loads
+ * it into the command line in place of the '--argsfile' argument. By default the args file
+ * will be loaded if it exists, and a warning will be given if it does not.
+ *
+ * The '--argsfile-required &lt;somepath&gt;' version will throw an error if the args file
+ * is not present, but it will not report any warnings or details otherwise.
+ *
+ * The `--argsfile-optional &lt;somepath&gt; version will not throw an error if the args
+ * file is not present, and it will not report any warnings or details otherwise.
+ *
+ * A prefix command line can be given to ArgsFile to pre-load any settings. In this way
+ * it is possible to easily provide a default args file which will be loaded. For example,
+ * A prefix command of '--argsfile-optional &lt;somepath&gt;' will load options if they are
+ * available in the specified file, but will otherwise provide no feedback to the user.
+ *
+ * <H1>ArgsFile Injection</H1>
+ *
+ * When an argsfile is loaded, it reads a command from each line into the current position
+ * of the command line. No parsing is done. Blank lines are ignored. Newlines are used as the
+ * argument delimiter, and lines that end with a backslash before the newline are automatically
+ * joined together.
+ *
+ * <H1>ArgsFile Diagnostics</H1>
+ *
+ * All modifications to the command line should be reported to the logging facility at
+ * INFO level. This assumes that the calling layer wants to inform users of command line injections,
+ * and that the user can select to be notified of warnings only if desired.
+ *
+ * <H1>Environment Variables</H1>
+ *
+ * Simple environment variable substitution is attempted for any pattern which appears as '$' followed
+ * by all uppercase letters and underscores. Any references of this type which are not resolvable
+ * will cause an error to be thrown.
+ */
+public class NBCLIArgsFile {
+    private final static Logger logger = LoggerFactory.getLogger("ARGSFILE");
+
+    // Options which may contextualize other CLI options or commands.
+    // These must be parsed first
+    public static final String ARGS_FILE = "--argsfile";
+    public static final String ARGS_FILE_OPTIONAL = "--argsfile-optional";
+    public static final String ARGS_FILE_REQUIRED = "--argsfile-required";
+    public static final String ARGS_PIN = "--pin";
+    public static final String ARGS_UNPIN = "--unpin";
+
+    private Path argsPath;
+    private LinkedList<String> preload;
+    private final Set<String> stopWords = new HashSet<>();
+    private final LinkedHashSet<String> args = new LinkedHashSet<>();
+    LinkedHashSet<String> argsToPin = new LinkedHashSet<>();
+    LinkedHashSet<String> argsToUnpin = new LinkedHashSet<>();
+    private final Set<String> readPaths = new HashSet<>();
+
+    public NBCLIArgsFile() {
+    }
+
+    public NBCLIArgsFile preload(String... preload) {
+        this.preload = new LinkedList<String>(Arrays.asList(preload));
+        return this;
+    }
+
+    /**
+     * Indicate which words are invalid for the purposes of matching
+     * trailing parts of arguments. The provided words will not
+     * be considered as valid values to arguments in any case.
+     *
+     * @param reservedWords Words to ignore in option values
+     * @return this ArgsFile, for method chaining
+     */
+    public NBCLIArgsFile reserved(Collection<String> reservedWords) {
+        this.stopWords.addAll(reservedWords);
+        return this;
+    }
+
+    /**
+     * Indicate which words are invalid for the purposes of matching
+     * trailing parts of arguments. The provided words will not
+     * be considered as valid values to arguments in any case.
+     *
+     * @param reservedWords Words to ignore in option values
+     * @return this ArgsFile, for method chaining
+     */
+    public NBCLIArgsFile reserved(String... reservedWords) {
+        this.stopWords.addAll(Arrays.asList(reservedWords));
+        return this;
+    }
+
+    private enum Selection {
+        // Ignore if not present, show injections at info
+        IgnoreIfMissing,
+        // Warn if not present, but show injections at info
+        WarnIfMissing,
+        // throw error if not present, show injections at info
+        ErrorIfMissing
+    }
+
+    public LinkedList<String> process(String... args) {
+        return process(new LinkedList<String>(Arrays.asList(args)));
+    }
+
+    public LinkedList<String> process(LinkedList<String> commandline) {
+        if (preload != null) {
+            LinkedList<String> modified = new LinkedList<String>();
+            modified.addAll(preload);
+            modified.addAll(commandline);
+            preload = null;
+            commandline = modified;
+        }
+        LinkedList<String> composed = new LinkedList<>();
+
+        while (commandline.peekFirst() != null) {
+            String arg = commandline.peekFirst();
+            switch (arg) {
+                case ARGS_FILE:
+                    pinAndUnpin();
+                    commandline.removeFirst();
+                    String argspath = readWordOrThrow(commandline, "path to an args file");
+                    setArgsFile(argspath, Selection.WarnIfMissing);
+                    commandline = mergeArgs(this.argsPath, Selection.WarnIfMissing, commandline);
+                    break;
+                case ARGS_FILE_REQUIRED:
+                    commandline.removeFirst();
+                    String argspathRequired = readWordOrThrow(commandline, "path to an args file");
+                    setArgsFile(argspathRequired, Selection.ErrorIfMissing);
+                    commandline = mergeArgs(this.argsPath, Selection.ErrorIfMissing, commandline);
+                    break;
+                case ARGS_FILE_OPTIONAL:
+                    commandline.removeFirst();
+                    String argspathOptional = readWordOrThrow(commandline, "path to an args file");
+                    setArgsFile(argspathOptional, Selection.IgnoreIfMissing);
+                    commandline = mergeArgs(this.argsPath, Selection.IgnoreIfMissing, commandline);
+                    break;
+                case ARGS_PIN:
+                    commandline.removeFirst();
+                    argsToPin.addAll(argsToLines(readOptionAndArg(commandline, false)));
+                    break;
+                case ARGS_UNPIN:
+                    commandline.removeFirst();
+                    argsToUnpin.addAll(argsToLines(readOptionAndArg(commandline, true)));
+                    break;
+                default:
+                    composed.addLast(commandline.removeFirst());
+            }
+        }
+        pinAndUnpin();
+        return composed;
+    }
+
+    private void pinAndUnpin() {
+        if (this.argsToUnpin.size() == 0 && this.argsToPin.size() == 0) {
+            return;
+        }
+        LinkedHashSet<String> extant = readArgsFile(this.argsPath, Selection.IgnoreIfMissing);
+        LinkedHashSet<String> mergedPins = mergePins(this.argsToPin, this.argsToUnpin, extant);
+        if (extant.equals(mergedPins)) {
+            logger.info("Pinning resulted in no changes to argsfile '" + this.argsPath.toString() + "'");
+        } else {
+            logger.info("Writing updated argsfile '" + this.argsPath.toString() + "' with " +
+                    (this.argsToPin.size() + this.argsToUnpin.size()) + " changes");
+            writeArgsFile(mergedPins);
+        }
+
+        this.argsToPin.clear();
+        this.argsToUnpin.clear();
+
+    }
+
+
+    private LinkedHashSet<String> mergePins(
+            LinkedHashSet<String> toPin,
+            LinkedHashSet<String> toUnpin,
+            LinkedHashSet<String> extant) {
+
+        LinkedHashSet<String> merged = new LinkedHashSet<>();
+        merged.addAll(extant);
+
+        for (String arg : toPin) {
+            if (argsToUnpin.contains(arg)) {
+                throw new RuntimeException("You have both --pin and --unpin for '" + arg + ", I don't know which " +
+                        "one you want.");
+            }
+        }
+        for (String arg : toUnpin) {
+            if (argsToPin.contains(arg)) {
+                throw new RuntimeException("You have both --pin and --unpin for '" + arg + ", I don't know which " +
+                        "one you want.");
+            }
+        }
+
+        for (String toAdd : toPin) {
+            if (merged.contains(toAdd)) {
+                logger.warn("Requested to pin argument again: '" + toAdd + "', ignoring");
+            } else {
+                logger.info("Pinning option '" + toAdd + "' to '" + this.argsPath.toString() + "'");
+                merged.add(toAdd);
+            }
+        }
+
+        for (String toDel : toUnpin) {
+            if (merged.contains(toDel)) {
+                logger.info("Unpinning '" + toDel + "' from '" + this.argsPath.toString() + "'");
+                merged.remove(toDel);
+            } else {
+                logger.warn("Requested to unpin argument '" + toDel + "' which was not found in " + argsPath.toString());
+            }
+        }
+
+        return merged;
+    }
+
+    LinkedList<String> mergeArgs(Path argspath, Selection mode, LinkedList<String> commandline) {
+        this.args.clear();
+        if (this.readPaths.contains(argsPath.toString())) {
+            throw new BasicError("Recursive reading of argsfile is detected for '" + argspath.toString() + "'.\n" +
+                    "Please ensure that you do not have cyclic references in your arguments for argsfiles.");
+        }
+        LinkedHashSet<String> loaded = readArgsFile(argspath, mode);
+        this.readPaths.add(argsPath.toString());
+
+        List<String> interpolated = loaded.stream()
+                .map(p -> {
+                    String q = Environment.INSTANCE.interpolate(p).orElse(p);
+                    if (!q.equals(p)) {
+                        logger.info("argsfile: '" + argsPath.toString() + "': loaded option '" + p + "' as '" + q + "'");
+                    }
+                    return q;
+                })
+                .collect(Collectors.toList());
+
+        LinkedList<String> inArgvForm = linesToArgs(interpolated);
+        this.args.addAll(inArgvForm);
+        return concat(inArgvForm, commandline);
+    }
+
+    private LinkedList<String> concat(Collection<String>... entries) {
+        LinkedList<String> composed = new LinkedList<>();
+        for (Collection<String> list : entries) {
+            composed.addAll(list);
+        }
+        return composed;
+    }
+
+    /**
+     * <p>Load the args file into an args array. The returned format follows
+     * the standard pattern of args as you would see for a main method, although
+     * the internal format is structured to support easy editing and clarity.</p>
+     *
+     * <p>
+     * The args file is stored in a simple option-per-line format which
+     * follows these rules:
+     * <UL>
+     * <LI>Lines ending with a backslash (\) only are concatenated to the next
+     * line with the backslash removed.
+     * <LI>Line content consists of one option and one optional argument.</LI>
+     * <LI>Options must start with at least one dash (-).</LI>
+     * <LI>If an argument is provided for an option, it follows the option and a space.</LI>
+     * <LI>Empty lines and lines which start with '//' or '#' are ignored.</LI>
+     * <LI>Lines which are identical after applying the above rules are elided
+     * down to the last occurence.</LI>
+     * </UL>
+     * </p>
+     *
+     * <p>
+     * This allows for multi-valued options, or options which can be specified multiple
+     * times with different arguments to be supported, so long as each occurrence has a
+     * unique option value.
+     * </p>
+     *
+     * @param argspath The path of the argsfile to load
+     * @param mode     The level of feedback to provide in the case of a missing file
+     * @return The argsfile content, structured like an args array
+     */
+    private LinkedHashSet<String> readArgsFile(Path argspath, Selection mode) {
+        LinkedHashSet<String> args = new LinkedHashSet<>();
+
+        if (!assertArgsFileExists(argspath, mode)) {
+            return args;
+        }
+
+        List<String> lines = null;
+        try {
+            lines = Files.readAllLines(argspath);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        List<String> content = lines.stream()
+                .filter(s -> !s.startsWith("#"))
+                .filter(s -> !s.startsWith("/"))
+                .filter(s -> !s.isBlank())
+                .filter(s -> !s.isEmpty())
+                .collect(Collectors.toList());
+        StringBuilder splitword = new StringBuilder();
+        LinkedHashSet<String> loaded = new LinkedHashSet<>();
+        for (String s : content) {
+            splitword.append(s);
+            if (!s.endsWith("\\")) {
+                loaded.add(splitword.toString());
+                splitword.setLength(0);
+            } else {
+                splitword.setLength(splitword.length() - 1);
+            }
+        }
+        if (splitword.length() > 0) {
+            throw new RuntimeException("unqualified line continuation for '" + splitword.toString() + "'");
+        }
+
+        return loaded;
+    }
+
+    /**
+     * Write the argsfile in the format specified by {@link #readArgsFile(Path, Selection)}
+     *
+     * This method requires that an argsFile has been set by a previous
+     * --argsfile or --argsfile-required or --argsfile-optional option.
+     *
+     * @param args The args to write in one-arg-per-line form
+     */
+    private void writeArgsFile(LinkedHashSet<String> args) {
+        if (this.argsPath == null) {
+            throw new RuntimeException("No argsfile has been selected before using the pin option.");
+        }
+
+        try {
+            Files.createDirectories(
+                    this.argsPath.getParent(),
+                    PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwx---"))
+            );
+            Files.write(this.argsPath, args);
+        } catch (IOException e) {
+            throw new BasicError("unable to write '" + this.argsPath + "': " + e.getMessage());
+        }
+    }
+
+    private boolean assertArgsFileExists(Path argspath, Selection mode) {
+        if (!Files.exists(argsPath)) {
+            switch (mode) {
+                case ErrorIfMissing:
+                    throw new RuntimeException("A required argsfile was specified, but it does not exist: '" + argspath + "'");
+                case WarnIfMissing:
+                    logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'");
+                case IgnoreIfMissing:
+            }
+            return false;
+        }
+        return true;
+    }
+
+    private void setArgsFile(String argspath, Selection mode) {
+        Path selected = null;
+        String[] possibles = argspath.split(":");
+        for (String possible : possibles) {
+            Optional<String> expanded = Environment.INSTANCE.interpolate(possible);
+            if (expanded.isPresent()) {
+                Path possiblePath = Path.of(expanded.get());
+                if (Files.exists(possiblePath)) {
+                    selected = possiblePath;
+                    break;
+                }
+            }
+        }
+
+        if (selected == null) {
+            String defaultFirst = possibles[0];
+            defaultFirst = Environment.INSTANCE.interpolate(defaultFirst)
+                    .orElseThrow(() -> new RuntimeException("Invalid default argsfile: '" + possibles[0] + "'"));
+            selected = Path.of(defaultFirst);
+        }
+
+        this.argsPath = selected;
+        logger.debug("argsfile path is now '" + this.argsPath.toString() + "'");
+    }
+
+    /**
+     * Convert argv arguments to consolidated form which is used in the args file.
+     * This means that options and their (optional) arguments are on the
+     * same line, concatenated with a space after the option.
+     *
+     * @return The arg-per-line form
+     */
+    LinkedHashSet<String> argsToLines(List<String> args) {
+        LinkedHashSet<String> lines = new LinkedHashSet<>();
+        Iterator<String> iter = args.iterator();
+        List<String> element = new ArrayList<>();
+        while (iter.hasNext()) {
+            String word = iter.next();
+            if (word.startsWith("-")) {
+                if (element.size() > 0) {
+                    lines.add(Strings.join(element, " "));
+                    element.clear();
+                }
+            }
+            element.add(word);
+        }
+        lines.add(Strings.join(element, " "));
+        return lines;
+    }
+
+    /**
+     * Convert arg lines as used in an args file to the argv which
+     * is used on the command line.
+     *
+     * @return The argv list as you would see with {@code main(String[] argv)}
+     */
+    LinkedList<String> linesToArgs(Collection<String> lines) {
+        LinkedList<String> args = new LinkedList<>();
+        for (String line : lines) {
+            if (line.startsWith("-")) {
+                String[] words = line.split(" ", 2);
+                args.addAll(Arrays.asList(words));
+            } else {
+                args.add(line);
+            }
+        }
+        return args;
+    }
+
+    private LinkedList<String> unpin(LinkedList<String> arglist) {
+        if (this.argsPath == null) {
+            throw new RuntimeException("No argsfile has been selected before using the pin option.");
+        }
+        return arglist;
+    }
+
+    /**
+     * Read the current command line option from the argument list,
+     * so long as it is a dash or double-dash option, and is not a
+     * reserved word, and any argument that goes with it, if any.
+     *
+     * @param arglist The command line containing the option
+     * @return A list containing the current command line option
+     */
+    private LinkedList<String> readOptionAndArg(LinkedList<String> arglist, boolean consume) {
+        LinkedList<String> option = new LinkedList<>();
+        ListIterator<String> iter = arglist.listIterator();
+
+        if (!iter.hasNext()) {
+            throw new RuntimeException("Arguments must follow the --pin option");
+        }
+        String opt = iter.next();
+
+        if (!opt.startsWith("-") || stopWords.contains(opt)) {
+            throw new RuntimeException("Arguments following the --pin option must not" +
+                    " be commands like '" + opt + "'");
+        }
+        option.add(opt);
+        if (consume) {
+            iter.remove();
+        }
+
+        if (iter.hasNext()) {
+            opt = iter.next();
+            if (!stopWords.contains(opt) && !opt.startsWith("-")) {
+                option.add(opt);
+                if (consume) {
+                    iter.remove();
+                }
+            }
+        }
+        return option;
+    }
+
+
+    /**
+     * Consume the next word from the beginning of the linked list. If there is
+     * no word to consume, then throw an error with the description.
+     *
+     * @param commandline A list of words
+     * @param description A description of what the next word value is meant to represent
+     * @return The next word from the list
+     */
+    private String readWordOrThrow(LinkedList<String> commandline, String description) {
+        String found = commandline.peekFirst();
+        if (found == null) {
+            throw new RuntimeException("Unable to read argument top option for " + description);
+        }
+        return commandline.removeFirst();
+    }
+}

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -4,11 +4,15 @@ import ch.qos.logback.classic.Level;
 import io.nosqlbench.engine.api.metrics.IndicatorMode;
 import io.nosqlbench.engine.api.util.Unit;
 import io.nosqlbench.engine.core.script.Scenario;
+import io.nosqlbench.nb.api.errors.BasicError;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
 import java.security.InvalidParameterException;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -19,19 +23,16 @@ import java.util.stream.Collectors;
  */
 public class NBCLIOptions {
 
-    private final static String userHome = System.getProperty("user.home");
+    private final static Logger logger = LoggerFactory.getLogger("OPTIONS");
-    private final static Path defaultOptFile = Path.of(userHome, ".nosqlbench/options");
-
-    private final static Logger logger = LoggerFactory.getLogger(NBCLIOptions.class);
-
-    // Options which may contextualize other CLI options or commands.
-    // These must be parsed first
-    private static final String ARGS_FILE = "--argsfile";
-    private static final String ARGS_FILE_DEFAULT = "$HOME/.nosqlbench/argsfile";
-    private static final String ARGS_PIN = "--pin";
-    private static final String ARGS_UNPIN = "--unpin";
 
+    private final static String NB_STATE_DIR = "--statedir";
+    private final static String NB_STATEDIR_PATHS = "$NBSTATEDIR:$PWD/.nosqlbench:$HOME/.nosqlbench";
+    public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile";
     private static final String INCLUDE = "--include";
+
+    private final static String userHome = System.getProperty("user.home");
+
+
     private static final String METRICS_PREFIX = "--metrics-prefix";
 
     //    private static final String ANNOTATE_TO_GRAFANA = "--grafana-baseurl";
@@ -39,7 +40,6 @@ public class NBCLIOptions {
     private static final String ANNOTATORS_CONFIG = "--annotators";
     private static final String DEFAULT_ANNOTATORS = "all";
 
-
     // Discovery
     private static final String HELP = "--help";
     private static final String LIST_METRICS = "--list-metrics";
@@ -91,6 +91,7 @@ public class NBCLIOptions {
     private static final String DOCKER_GRAFANA_TAG = "--docker-grafana-tag";
 
     private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
+    public static final String NBSTATEDIR = "NBSTATEDIR";
 
     private final LinkedList<Cmd> cmdList = new LinkedList<>();
     private int logsMax = 0;
@@ -130,13 +131,16 @@ public class NBCLIOptions {
     private Scenario.Engine engine = Scenario.Engine.Graalvm;
     private boolean graaljs_compat = false;
     private int hdr_digits = 4;
-    private String docker_grafana_tag = "7.0.1";
+    private String docker_grafana_tag = "7.2.2";
     private boolean showStackTraces = false;
     private boolean compileScript = false;
     private String scriptFile = null;
     private String[] annotateEvents = new String[]{"ALL"};
     private String dockerMetricsHost;
     private String annotatorsConfig = "";
+    private String statedirs = NB_STATEDIR_PATHS;
+    private Path statepath;
+    private List<String> statePathAccesses = new ArrayList<>();
 
     public String getAnnotatorsConfig() {
         return annotatorsConfig;
@@ -163,8 +167,6 @@ public class NBCLIOptions {
     }
 
     private LinkedList<String> parseGlobalOptions(String[] args) {
-        ArgsFile argsfile = new ArgsFile();
-        argsfile.preload("--argsfile-optional", ARGS_FILE_DEFAULT);
 
         LinkedList<String> arglist = new LinkedList<>() {{
             addAll(Arrays.asList(args));
@@ -175,7 +177,8 @@ public class NBCLIOptions {
             return arglist;
         }
 
-        // Preprocess --include regardless of position
+        // Process --include and --statedir, separately first
+        // regardless of position
         LinkedList<String> nonincludes = new LinkedList<>();
         while (arglist.peekFirst() != null) {
             String word = arglist.peekFirst();
@@ -188,9 +191,53 @@ public class NBCLIOptions {
             }
 
             switch (word) {
-                case ARGS_FILE:
+                case NB_STATE_DIR:
-                case ARGS_PIN:
+                    arglist.removeFirst();
-                case ARGS_UNPIN:
+                    this.statedirs = readWordOrThrow(arglist, "nosqlbench global state directory");
+                    break;
+                case INCLUDE:
+                    arglist.removeFirst();
+                    String include = readWordOrThrow(arglist, "path to include");
+                    wantsToIncludePaths.add(include);
+                    break;
+                default:
+                    nonincludes.addLast(arglist.removeFirst());
+            }
+        }
+        this.statedirs = (this.statedirs != null ? this.statedirs : NB_STATEDIR_PATHS);
+        this.setStatePath();
+
+        arglist = nonincludes;
+        nonincludes = new LinkedList<>();
+
+        // Now that statdirs is settled, auto load argsfile if it is present
+        NBCLIArgsFile argsfile = new NBCLIArgsFile();
+        argsfile.reserved(NBCLICommandParser.RESERVED_WORDS);
+        argsfile.preload("--argsfile-optional", ARGS_FILE_DEFAULT);
+        arglist = argsfile.process(arglist);
+
+        // Parse all --argsfile... and other high level options
+
+        while (arglist.peekFirst() != null) {
+            String word = arglist.peekFirst();
+            if (word.startsWith("--") && word.contains("=")) {
+                String wordToSplit = arglist.removeFirst();
+                String[] split = wordToSplit.split("=", 2);
+                arglist.offerFirst(split[1]);
+                arglist.offerFirst(split[0]);
+                continue;
+            }
+
+            switch (word) {
+                // These options modify other options. They should be processed early.
+                case NBCLIArgsFile.ARGS_FILE:
+                case NBCLIArgsFile.ARGS_FILE_OPTIONAL:
+                case NBCLIArgsFile.ARGS_FILE_REQUIRED:
+                case NBCLIArgsFile.ARGS_PIN:
+                case NBCLIArgsFile.ARGS_UNPIN:
+                    if (this.statepath == null) {
+                        setStatePath();
+                    }
                     arglist = argsfile.process(arglist);
                     break;
                 case ANNOTATE_EVENTS:
@@ -198,19 +245,10 @@ public class NBCLIOptions {
                     String toAnnotate = readWordOrThrow(arglist, "annotated events");
                     annotateEvents = toAnnotate.split("\\\\s*,\\\\s*");
                     break;
-//                case ANNOTATE_TO_GRAFANA:
-//                    arglist.removeFirst();
-//                    grafanaEndpoint = readWordOrThrow(arglist,"grafana API endpoint");
-//                    break;
                 case ANNOTATORS_CONFIG:
                     arglist.removeFirst();
                     this.annotatorsConfig = readWordOrThrow(arglist, "annotators config");
                     break;
-                case INCLUDE:
-                    arglist.removeFirst();
-                    String include = readWordOrThrow(arglist, "path to include");
-                    wantsToIncludePaths.add(include);
-                    break;
                 case REPORT_GRAPHITE_TO:
                     arglist.removeFirst();
                     reportGraphiteTo = arglist.removeFirst();
@@ -245,13 +283,57 @@ public class NBCLIOptions {
                     break;
                 default:
                     nonincludes.addLast(arglist.removeFirst());
-
+            }
         }
 
-        }
         return nonincludes;
     }
 
+    private Path setStatePath() {
+        if (statePathAccesses.size() > 0) {
+            throw new BasicError("The statedir must be set before it is used by other\n" +
+                    " options. If you want to change the statedir, be sure you do it before\n" +
+                    " dependent options. These parameters were called before this --statedir:\n" +
+                    statePathAccesses.stream().map(s -> "> " + s).collect(Collectors.joining("\n")));
+        }
+        if (this.statepath != null) {
+            return this.statepath;
+        }
+
+        List<String> paths = Environment.INSTANCE.interpolate(":", statedirs);
+        Path selected = null;
+
+        for (String pathName : paths) {
+            Path path = Path.of(pathName);
+            if (Files.exists(path)) {
+                if (Files.isDirectory(path)) {
+                    selected = path;
+                    break;
+                } else {
+                    logger.warn("possible state dir path is not a directory: '" + path.toString() + "'");
+                }
+            }
+        }
+        if (selected == null) {
+            selected = Path.of(paths.get(0));
+        }
+
+        if (!Files.exists(selected)) {
+            try {
+                Files.createDirectories(
+                        selected,
+                        PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwx---"))
+                );
+            } catch (IOException e) {
+                throw new BasicError("Could not create state directory at '" + selected.toString() + "': " + e.getMessage());
+            }
+        }
+
+        Environment.INSTANCE.put(NBSTATEDIR, selected.toString());
+
+        return selected;
+    }
+
     private void parseAllOptions(String[] args) {
         LinkedList<String> arglist = parseGlobalOptions(args);
 

engine-cli/src/test/java/io/nosqlbench/engine/cli/ArgsFileTest.java

@@ -1,40 +0,0 @@
-package io.nosqlbench.engine.cli;
-
-import org.junit.Test;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class ArgsFileTest {
-
-    @Test
-    public void testLoadingArgs() {
-        LinkedList<String> result;
-        ArgsFile argsFile = new ArgsFile();
-        result = argsFile.process("--argsfile", "src/test/resources/argsfiles/nonextant.cli");
-        assertThat(result).containsExactly();
-        result = argsFile.process("--argsfile", "src/test/resources/argsfiles/alphagamma.cli");
-        assertThat(result).containsExactly("alpha", "gamma");
-    }
-
-    @Test(expected = RuntimeException.class)
-    public void testLoadingMissingRequiredFails() {
-        LinkedList<String> result;
-        ArgsFile argsFile = new ArgsFile();
-        result = argsFile.process("--argsfile-required", "src/test/resources/argsfiles/nonextant.cli");
-    }
-
-    @Test
-    public void testLoadingInPlace() {
-        LinkedList<String> result;
-        LinkedList<String> commands = new LinkedList<>(List.of("--abc", "--def", "--argsfile", "src/test/resources" +
-                "/argsfiles/alphagamma.cli"));
-        ArgsFile argsFile = new ArgsFile().preload("--argsfile-optional", "src/test/resources/argsfiles/alphagamma" +
-                ".cli");
-        result = argsFile.process(commands);
-        assertThat(result).containsExactly("alpha", "gamma", "--abc", "--def", "alpha", "gamma");
-    }
-
-}

engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIArgsFileTest.java

@@ -0,0 +1,117 @@
+package io.nosqlbench.engine.cli;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class NBCLIArgsFileTest {
+
+    @Test
+    public void testLoadingArgs() {
+        LinkedList<String> result;
+        NBCLIArgsFile argsFile = new NBCLIArgsFile();
+        result = argsFile.process("--argsfile", "src/test/resources/argsfiles/nonextant.cli");
+        assertThat(result).containsExactly();
+        result = argsFile.process("--argsfile", "src/test/resources/argsfiles/alphagamma.cli");
+        assertThat(result).containsExactly("alpha", "gamma");
+    }
+
+    @Test
+    public void loadParamsWithEnv() {
+        NBCLIArgsFile argsfile = new NBCLIArgsFile();
+        LinkedList<String> result = argsfile.process("--argsfile-required", "src/test/resources/argsfiles/home_env.cli");
+        System.out.println(result);
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testLoadingMissingRequiredFails() {
+        LinkedList<String> result;
+        NBCLIArgsFile argsFile = new NBCLIArgsFile();
+        result = argsFile.process("--argsfile-required", "src/test/resources/argsfiles/nonextant.cli");
+    }
+
+    @Test
+    public void testLoadingInPlace() {
+        LinkedList<String> result;
+        LinkedList<String> commands = new LinkedList<>(List.of("--abc", "--def"));
+
+        NBCLIArgsFile argsFile = new NBCLIArgsFile().preload("--argsfile-optional", "src/test/resources/argsfiles/alphagamma" +
+                ".cli");
+        result = argsFile.process(commands);
+        assertThat(result).containsExactly("alpha", "gamma", "--abc", "--def");
+    }
+
+    @Test
+    public void testLinesToArgs() {
+        NBCLIArgsFile argsfile = new NBCLIArgsFile().reserved("reservedword");
+        LinkedList<String> args = argsfile.linesToArgs(
+                List.of("--optionname argument", "--optionname2", "--opt3 reservedword")
+        );
+        assertThat(args).containsExactly("--optionname", "argument", "--optionname2", "--opt3", "reservedword");
+    }
+
+    @Test
+    public void testOptionPinning() {
+        Path tempFile = null;
+        try {
+            tempFile = Files.createTempFile(
+                    "tmpfile",
+                    "cli",
+                    PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-rw-r--"))
+            );
+
+            // preloading is a way to have global defaults based on
+            // the presence of a default file
+            NBCLIArgsFile argsfile = new NBCLIArgsFile().preload(
+                    "--argsfile-optional", tempFile.toAbsolutePath().toString()
+            );
+
+            LinkedList<String> commandline;
+
+            commandline = argsfile.process(
+                    "--pin", "--option1",
+                    "--pin", "--option1",
+                    "--pin", "--option2", "arg2"
+            );
+
+            String filecontents;
+            filecontents = Files.readString(tempFile);
+
+            // logging should indicate no changes
+            commandline = argsfile.process(
+                    "--pin", "--option1",
+                    "--pin", "--option1",
+                    "--pin", "--option2", "arg2"
+            );
+
+            // unpinned options should be discarded
+            commandline = argsfile.process(
+                    "--unpin", "--option1",
+                    "--option4", "--option5"
+            );
+
+            assertThat(commandline).containsExactly(
+                    "--option4", "--option5"
+            );
+
+            assertThat(filecontents).isEqualTo("--option1\n--option2 arg2\n");
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        } finally {
+            try {
+                Files.deleteIfExists(tempFile);
+            } catch (Exception ignored) {
+            }
+        }
+    }
+
+
+}