From 1382d33d12a4c396d87b7bd7d9eb993b3060a472 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 29 Oct 2020 10:55:38 -0500 Subject: [PATCH] argsfiles and statedirs --- .../io/nosqlbench/engine/cli/ArgsFile.java | 234 -------- .../java/io/nosqlbench/engine/cli/NBCLI.java | 64 +-- .../nosqlbench/engine/cli/NBCLIArgsFile.java | 500 ++++++++++++++++++ .../nosqlbench/engine/cli/NBCLIOptions.java | 152 ++++-- .../nosqlbench/engine/cli/ArgsFileTest.java | 40 -- .../engine/cli/NBCLIArgsFileTest.java | 117 ++++ 6 files changed, 768 insertions(+), 339 deletions(-) delete mode 100644 engine-cli/src/main/java/io/nosqlbench/engine/cli/ArgsFile.java create mode 100644 engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIArgsFile.java delete mode 100644 engine-cli/src/test/java/io/nosqlbench/engine/cli/ArgsFileTest.java create mode 100644 engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIArgsFileTest.java diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/ArgsFile.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/ArgsFile.java deleted file mode 100644 index 94dcd4258..000000000 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/ArgsFile.java +++ /dev/null @@ -1,234 +0,0 @@ -package io.nosqlbench.engine.cli; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - *

Synopsis

- * - * ArgsFile is a command-line modifier which can take linked list of - * command args and modify it, and/or modify argsfile refrenced in this way. - * - *

ArgsFile Selection

- * - * During processing, any occurence of '--argsfile' selects the active argsfile and loads - * it into the command line in place of the '--argsfile' argument. By default the args file - * will be loaded if it exists, and a warning will be given if it does not. - * - * The '--argsfile-required <somepath>' version will throw an error if the args file - * is not present, but it will not report any warnings or details otherwise. - * - * The `--argsfile-optional <somepath> version will not throw an error if the args - * file is not present, and it will not report any warnings or details otherwise. - * - * A prefix command line can be given to ArgsFile to pre-load any settings. In this way - * it is possible to easily provide a default args file which will be loaded. For example, - * A prefix command of '--argsfile-optional <somepath>' will load options if they are - * available in the specified file, but will otherwise provide no feedback to the user. - * - *

ArgsFile Injection

- * - * When an argsfile is loaded, it reads a command from each line into the current position - * of the command line. No parsing is done. Blank lines are ignored. Newlines are used as the - * argument delimiter, and lines that end with a backslash before the newline are automatically - * joined together. - * - *

ArgsFile Diagnostics

- * - * All modifications to the command line should be reported to the logging facility at - * INFO level. This assumes that the calling layer wants to inform users of command line injections, - * and that the user can select to be notified of warnings only if desired. - * - *

Environment Variables

- * - * Simple environment variable substitution is attempted for any pattern which appears as '$' followed - * by all uppercase letters and underscores. Any references of this type which are not resolvable - * will cause an error to be thrown. - */ -public class ArgsFile { - private final static Logger logger = LoggerFactory.getLogger(ArgsFile.class); - - private Path argsPath; - private LinkedList preload; - - public ArgsFile() { - } - - public ArgsFile preload(String... preload) { - this.preload = new LinkedList(Arrays.asList(preload)); - return this; - } - - private enum Selection { - // Ignore if not present, show injections at info - IgnoreIfMissing, - // Warn if not present, but show injections at info - WarnIfMissing, - // throw error if not present, show injections at info - ErrorIfMissing - } - - public LinkedList process(String... args) { - return process(new LinkedList(Arrays.asList(args))); - } - - public LinkedList process(LinkedList commandline) { - if (preload != null) { - LinkedList modified = new LinkedList(); - modified.addAll(preload); - modified.addAll(commandline); - preload = null; - commandline = modified; - } - LinkedList composed = new LinkedList<>(); - while (commandline.peekFirst() != null) { - String arg = commandline.peekFirst(); - switch (arg) { - case "--argsfile": - commandline.removeFirst(); - String argspath = readWordOrThrow(commandline, "path to an args file"); - setArgsFile(argspath, Selection.WarnIfMissing); - commandline = loadArgs(this.argsPath, Selection.WarnIfMissing, commandline); - break; - case "--argsfile-required": - commandline.removeFirst(); - String argspathRequired = readWordOrThrow(commandline, "path to an args file"); - setArgsFile(argspathRequired, Selection.ErrorIfMissing); - commandline = loadArgs(this.argsPath, Selection.ErrorIfMissing, commandline); - break; - case "--argsfile-optional": - commandline.removeFirst(); - String argspathOptional = readWordOrThrow(commandline, "path to an args file"); - setArgsFile(argspathOptional, Selection.IgnoreIfMissing); - commandline = loadArgs(this.argsPath, Selection.IgnoreIfMissing, commandline); - break; - case "--pin": - commandline.removeFirst(); - commandline = pinArg(commandline); - break; - case "--unpin": - commandline.removeFirst(); - commandline = unpinArg(commandline); - break; - default: - composed.addLast(commandline.removeFirst()); - } - - } - return composed; - } - - private LinkedList loadArgs(Path argspath, Selection mode, LinkedList commandline) { - if (!assertArgsFileExists(argspath, mode)) { - return commandline; - } - List lines = null; - try { - lines = Files.readAllLines(argspath); - } catch (IOException e) { - throw new RuntimeException(e); - } - List content = lines.stream() - .filter(s -> !s.startsWith("#")) - .filter(s -> !s.startsWith("/")) - .filter(s -> !s.isBlank()) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList()); - StringBuilder splitword = new StringBuilder(); - LinkedList loaded = new LinkedList<>(); - for (String s : content) { - splitword.append(s); - if (!s.endsWith("\\")) { - loaded.addLast(splitword.toString()); - splitword.setLength(0); - } else { - splitword.setLength(splitword.length() - 1); - } - } - if (splitword.length() > 0) { - throw new RuntimeException("unqualified line continuation for '" + splitword.toString() + "'"); - } - - Iterator injections = loaded.descendingIterator(); - while (injections.hasNext()) { - String injection = injections.next(); - injection = injectEnv(injection); - commandline.addFirst(injection); - } - - return commandline; - } - - private boolean assertArgsFileExists(Path argspath, Selection mode) { - if (!Files.exists(argsPath)) { - switch (mode) { - case ErrorIfMissing: - throw new RuntimeException("A required argsfile was specified, but it does not exist: '" + argspath + "'"); - case WarnIfMissing: - logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'"); - case IgnoreIfMissing: - } - return false; - } - return true; - } - - private void setArgsFile(String argspath, Selection mode) { - this.argsPath = Path.of(argspath); -// assertIfMissing(this.argsPath,mode); - } - - private String readWordOrThrow(LinkedList commandline, String description) { - String found = commandline.peekFirst(); - if (found == null) { - throw new RuntimeException("Unable to read argument top option for " + description); - } - return commandline.removeFirst(); - } - - private LinkedList pinArg(LinkedList commandline) { - if (this.argsPath == null) { - throw new RuntimeException("No argsfile has been selected before using the pin option."); - } - return commandline; - } - - private LinkedList unpinArg(LinkedList commandline) { - if (this.argsPath == null) { - throw new RuntimeException("No argsfile has been selected before using the unpin option."); - } - return commandline; - } - - private String injectEnv(String word) { - Pattern envpattern = Pattern.compile("(?\\$[A-Z_]+)"); - Matcher matcher = envpattern.matcher(word); - StringBuilder sb = new StringBuilder(); - while (matcher.find()) { - String envvar = matcher.group("envvar"); - String value = System.getenv(envvar); - if (value == null) { - throw new RuntimeException("Env var '" + envvar + "' was not found in the environment."); - } - matcher.appendReplacement(sb, value); - } - matcher.appendTail(sb); - return sb.toString(); - } - - public LinkedList pin(LinkedList arglist) { - return arglist; - } - - public LinkedList unpin(LinkedList arglist) { - return arglist; - } -} diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 57caa129d..db6f08530 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -39,10 +39,10 @@ import java.util.stream.Collectors; public class NBCLI { - private static final Logger logger = LoggerFactory.getLogger(NBCLI.class); + private static final Logger logger = LoggerFactory.getLogger("NBCLI"); private static final Logger EVENTS = LoggerFactory.getLogger("EVENTS"); - private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log"; + private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log"; private final String commandName; @@ -70,36 +70,43 @@ public class NBCLI { NBCLIOptions globalOptions = new NBCLIOptions(args, NBCLIOptions.Mode.ParseGlobalsOnly); // Global only processing + if (args.length == 0) { + System.out.println(loadHelpFile("commandline.md")); + System.exit(0); + } + boolean dockerMetrics = globalOptions.wantsDockerMetrics(); + String dockerMetricsAt = globalOptions.wantsDockerMetricsAt(); String reportGraphiteTo = globalOptions.wantsReportGraphiteTo(); + int mOpts = (dockerMetrics ? 1 : 0) + (dockerMetricsAt != null ? 1 : 0) + (reportGraphiteTo != null ? 1 : 0); + if (mOpts > 1) { + throw new BasicError("You have multiple conflicting options which attempt to set\n" + + " the destination for metrics and annotations. Please select only one of\n" + + " --docker-metrics, --docker-metrics-at , or --report-graphite-to \n" + + " For more details, see run 'nb help docker-metrics'"); + } - if (globalOptions.wantsDockerMetrics()) { + String metricsAddr = null; + + if (dockerMetrics) { + // Setup docker stack for local docker metrics logger.info("Docker metrics is enabled. Docker must be installed for this to work"); DockerMetricsManager dmh = new DockerMetricsManager(); Map dashboardOptions = Map.of( - DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag() + DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag() ); dmh.startMetrics(dashboardOptions); - String warn = "Docker Containers are started, for grafana and prometheus, hit" + - " these urls in your browser: http://:3000 and http://:9090"; + " these urls in your browser: http://:3000 and http://:9090"; logger.warn(warn); - if (reportGraphiteTo != null) { - logger.warn(String.format("Docker metrics are enabled (--docker-metrics)" + - " but graphite reporting (--report-graphite-to) is set to %s \n" + - "usually only one of the two is configured.", - reportGraphiteTo)); - } else { - logger.info("Setting graphite reporting to localhost"); - reportGraphiteTo = "localhost:9109"; - } + metricsAddr = "localhost"; + } else if (dockerMetricsAt != null) { + metricsAddr = dockerMetricsAt; + } - if (globalOptions.getAnnotatorsConfig() != null) { - logger.warn("Docker metrics and separate annotations" + - "are configured (both --docker-metrics and --annotations)."); - } else { - Annotators.init("grafana{http://localhost:3000/}"); - } + if (metricsAddr != null) { + reportGraphiteTo = metricsAddr + ":9109"; + Annotators.init("{type:'grafana',url:'http://" + metricsAddr + ":3000/'}"); } if (args.length > 0 && args[0].toLowerCase().equals("virtdata")) { @@ -275,13 +282,6 @@ public class NBCLI { // intentionally not shown for warn-only logger.info("console logging level is " + options.wantsConsoleLogLevel()); - if (options.getCommands(). - - size() == 0) { - System.out.println(loadHelpFile("commandline.md")); - System.exit(0); - } - ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1); Scenario scenario = new Scenario( @@ -311,8 +311,6 @@ public class NBCLI { } - // Execute Scenario! - Level consoleLogLevel = options.wantsConsoleLogLevel(); Level scenarioLogLevel = Level.toLevel(options.getLogsLevel()); if (scenarioLogLevel.toInt() > consoleLogLevel.toInt()) { @@ -321,6 +319,12 @@ public class NBCLI { Level maxLevel = Level.toLevel(Math.min(consoleLogLevel.toInt(), scenarioLogLevel.toInt())); + // Execute Scenario! + if (options.getCommands().size() == 0) { + logger.info("No commands provided. Exiting before scenario."); + System.exit(0); + } + scenario.addScriptText(scriptData); ScriptParams scriptParams = new ScriptParams(); scriptParams.putAll(buffer.getCombinedParams()); diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIArgsFile.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIArgsFile.java new file mode 100644 index 000000000..ec047e94f --- /dev/null +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIArgsFile.java @@ -0,0 +1,500 @@ +package io.nosqlbench.engine.cli; + +import io.nosqlbench.nb.api.errors.BasicError; +import joptsimple.internal.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.*; +import java.util.stream.Collectors; + +/** + *

Synopsis

+ * + * ArgsFile is a command-line modifier which can take linked list of + * command args and modify it, and/or modify argsfile refrenced in this way. + * + *

ArgsFile Selection

+ * + * During processing, any occurence of '--argsfile' selects the active argsfile and loads + * it into the command line in place of the '--argsfile' argument. By default the args file + * will be loaded if it exists, and a warning will be given if it does not. + * + * The '--argsfile-required <somepath>' version will throw an error if the args file + * is not present, but it will not report any warnings or details otherwise. + * + * The `--argsfile-optional <somepath> version will not throw an error if the args + * file is not present, and it will not report any warnings or details otherwise. + * + * A prefix command line can be given to ArgsFile to pre-load any settings. In this way + * it is possible to easily provide a default args file which will be loaded. For example, + * A prefix command of '--argsfile-optional <somepath>' will load options if they are + * available in the specified file, but will otherwise provide no feedback to the user. + * + *

ArgsFile Injection

+ * + * When an argsfile is loaded, it reads a command from each line into the current position + * of the command line. No parsing is done. Blank lines are ignored. Newlines are used as the + * argument delimiter, and lines that end with a backslash before the newline are automatically + * joined together. + * + *

ArgsFile Diagnostics

+ * + * All modifications to the command line should be reported to the logging facility at + * INFO level. This assumes that the calling layer wants to inform users of command line injections, + * and that the user can select to be notified of warnings only if desired. + * + *

Environment Variables

+ * + * Simple environment variable substitution is attempted for any pattern which appears as '$' followed + * by all uppercase letters and underscores. Any references of this type which are not resolvable + * will cause an error to be thrown. + */ +public class NBCLIArgsFile { + private final static Logger logger = LoggerFactory.getLogger("ARGSFILE"); + + // Options which may contextualize other CLI options or commands. + // These must be parsed first + public static final String ARGS_FILE = "--argsfile"; + public static final String ARGS_FILE_OPTIONAL = "--argsfile-optional"; + public static final String ARGS_FILE_REQUIRED = "--argsfile-required"; + public static final String ARGS_PIN = "--pin"; + public static final String ARGS_UNPIN = "--unpin"; + + private Path argsPath; + private LinkedList preload; + private final Set stopWords = new HashSet<>(); + private final LinkedHashSet args = new LinkedHashSet<>(); + LinkedHashSet argsToPin = new LinkedHashSet<>(); + LinkedHashSet argsToUnpin = new LinkedHashSet<>(); + private final Set readPaths = new HashSet<>(); + + public NBCLIArgsFile() { + } + + public NBCLIArgsFile preload(String... preload) { + this.preload = new LinkedList(Arrays.asList(preload)); + return this; + } + + /** + * Indicate which words are invalid for the purposes of matching + * trailing parts of arguments. The provided words will not + * be considered as valid values to arguments in any case. + * + * @param reservedWords Words to ignore in option values + * @return this ArgsFile, for method chaining + */ + public NBCLIArgsFile reserved(Collection reservedWords) { + this.stopWords.addAll(reservedWords); + return this; + } + + /** + * Indicate which words are invalid for the purposes of matching + * trailing parts of arguments. The provided words will not + * be considered as valid values to arguments in any case. + * + * @param reservedWords Words to ignore in option values + * @return this ArgsFile, for method chaining + */ + public NBCLIArgsFile reserved(String... reservedWords) { + this.stopWords.addAll(Arrays.asList(reservedWords)); + return this; + } + + private enum Selection { + // Ignore if not present, show injections at info + IgnoreIfMissing, + // Warn if not present, but show injections at info + WarnIfMissing, + // throw error if not present, show injections at info + ErrorIfMissing + } + + public LinkedList process(String... args) { + return process(new LinkedList(Arrays.asList(args))); + } + + public LinkedList process(LinkedList commandline) { + if (preload != null) { + LinkedList modified = new LinkedList(); + modified.addAll(preload); + modified.addAll(commandline); + preload = null; + commandline = modified; + } + LinkedList composed = new LinkedList<>(); + + while (commandline.peekFirst() != null) { + String arg = commandline.peekFirst(); + switch (arg) { + case ARGS_FILE: + pinAndUnpin(); + commandline.removeFirst(); + String argspath = readWordOrThrow(commandline, "path to an args file"); + setArgsFile(argspath, Selection.WarnIfMissing); + commandline = mergeArgs(this.argsPath, Selection.WarnIfMissing, commandline); + break; + case ARGS_FILE_REQUIRED: + commandline.removeFirst(); + String argspathRequired = readWordOrThrow(commandline, "path to an args file"); + setArgsFile(argspathRequired, Selection.ErrorIfMissing); + commandline = mergeArgs(this.argsPath, Selection.ErrorIfMissing, commandline); + break; + case ARGS_FILE_OPTIONAL: + commandline.removeFirst(); + String argspathOptional = readWordOrThrow(commandline, "path to an args file"); + setArgsFile(argspathOptional, Selection.IgnoreIfMissing); + commandline = mergeArgs(this.argsPath, Selection.IgnoreIfMissing, commandline); + break; + case ARGS_PIN: + commandline.removeFirst(); + argsToPin.addAll(argsToLines(readOptionAndArg(commandline, false))); + break; + case ARGS_UNPIN: + commandline.removeFirst(); + argsToUnpin.addAll(argsToLines(readOptionAndArg(commandline, true))); + break; + default: + composed.addLast(commandline.removeFirst()); + } + } + pinAndUnpin(); + return composed; + } + + private void pinAndUnpin() { + if (this.argsToUnpin.size() == 0 && this.argsToPin.size() == 0) { + return; + } + LinkedHashSet extant = readArgsFile(this.argsPath, Selection.IgnoreIfMissing); + LinkedHashSet mergedPins = mergePins(this.argsToPin, this.argsToUnpin, extant); + if (extant.equals(mergedPins)) { + logger.info("Pinning resulted in no changes to argsfile '" + this.argsPath.toString() + "'"); + } else { + logger.info("Writing updated argsfile '" + this.argsPath.toString() + "' with " + + (this.argsToPin.size() + this.argsToUnpin.size()) + " changes"); + writeArgsFile(mergedPins); + } + + this.argsToPin.clear(); + this.argsToUnpin.clear(); + + } + + + private LinkedHashSet mergePins( + LinkedHashSet toPin, + LinkedHashSet toUnpin, + LinkedHashSet extant) { + + LinkedHashSet merged = new LinkedHashSet<>(); + merged.addAll(extant); + + for (String arg : toPin) { + if (argsToUnpin.contains(arg)) { + throw new RuntimeException("You have both --pin and --unpin for '" + arg + ", I don't know which " + + "one you want."); + } + } + for (String arg : toUnpin) { + if (argsToPin.contains(arg)) { + throw new RuntimeException("You have both --pin and --unpin for '" + arg + ", I don't know which " + + "one you want."); + } + } + + for (String toAdd : toPin) { + if (merged.contains(toAdd)) { + logger.warn("Requested to pin argument again: '" + toAdd + "', ignoring"); + } else { + logger.info("Pinning option '" + toAdd + "' to '" + this.argsPath.toString() + "'"); + merged.add(toAdd); + } + } + + for (String toDel : toUnpin) { + if (merged.contains(toDel)) { + logger.info("Unpinning '" + toDel + "' from '" + this.argsPath.toString() + "'"); + merged.remove(toDel); + } else { + logger.warn("Requested to unpin argument '" + toDel + "' which was not found in " + argsPath.toString()); + } + } + + return merged; + } + + LinkedList mergeArgs(Path argspath, Selection mode, LinkedList commandline) { + this.args.clear(); + if (this.readPaths.contains(argsPath.toString())) { + throw new BasicError("Recursive reading of argsfile is detected for '" + argspath.toString() + "'.\n" + + "Please ensure that you do not have cyclic references in your arguments for argsfiles."); + } + LinkedHashSet loaded = readArgsFile(argspath, mode); + this.readPaths.add(argsPath.toString()); + + List interpolated = loaded.stream() + .map(p -> { + String q = Environment.INSTANCE.interpolate(p).orElse(p); + if (!q.equals(p)) { + logger.info("argsfile: '" + argsPath.toString() + "': loaded option '" + p + "' as '" + q + "'"); + } + return q; + }) + .collect(Collectors.toList()); + + LinkedList inArgvForm = linesToArgs(interpolated); + this.args.addAll(inArgvForm); + return concat(inArgvForm, commandline); + } + + private LinkedList concat(Collection... entries) { + LinkedList composed = new LinkedList<>(); + for (Collection list : entries) { + composed.addAll(list); + } + return composed; + } + + /** + *

Load the args file into an args array. The returned format follows + * the standard pattern of args as you would see for a main method, although + * the internal format is structured to support easy editing and clarity.

+ * + *

+ * The args file is stored in a simple option-per-line format which + * follows these rules: + *

    + *
  • Lines ending with a backslash (\) only are concatenated to the next + * line with the backslash removed. + *
  • Line content consists of one option and one optional argument.
    • + *
  • Options must start with at least one dash (-).
    • + *
  • If an argument is provided for an option, it follows the option and a space.
    • + *
  • Empty lines and lines which start with '//' or '#' are ignored.
    • + *
  • Lines which are identical after applying the above rules are elided + * down to the last occurence.
    • + *
+ *

+ * + *

+ * This allows for multi-valued options, or options which can be specified multiple + * times with different arguments to be supported, so long as each occurrence has a + * unique option value. + *

+ * + * @param argspath The path of the argsfile to load + * @param mode The level of feedback to provide in the case of a missing file + * @return The argsfile content, structured like an args array + */ + private LinkedHashSet readArgsFile(Path argspath, Selection mode) { + LinkedHashSet args = new LinkedHashSet<>(); + + if (!assertArgsFileExists(argspath, mode)) { + return args; + } + + List lines = null; + try { + lines = Files.readAllLines(argspath); + } catch (IOException e) { + throw new RuntimeException(e); + } + List content = lines.stream() + .filter(s -> !s.startsWith("#")) + .filter(s -> !s.startsWith("/")) + .filter(s -> !s.isBlank()) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + StringBuilder splitword = new StringBuilder(); + LinkedHashSet loaded = new LinkedHashSet<>(); + for (String s : content) { + splitword.append(s); + if (!s.endsWith("\\")) { + loaded.add(splitword.toString()); + splitword.setLength(0); + } else { + splitword.setLength(splitword.length() - 1); + } + } + if (splitword.length() > 0) { + throw new RuntimeException("unqualified line continuation for '" + splitword.toString() + "'"); + } + + return loaded; + } + + /** + * Write the argsfile in the format specified by {@link #readArgsFile(Path, Selection)} + * + * This method requires that an argsFile has been set by a previous + * --argsfile or --argsfile-required or --argsfile-optional option. + * + * @param args The args to write in one-arg-per-line form + */ + private void writeArgsFile(LinkedHashSet args) { + if (this.argsPath == null) { + throw new RuntimeException("No argsfile has been selected before using the pin option."); + } + + try { + Files.createDirectories( + this.argsPath.getParent(), + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwx---")) + ); + Files.write(this.argsPath, args); + } catch (IOException e) { + throw new BasicError("unable to write '" + this.argsPath + "': " + e.getMessage()); + } + } + + private boolean assertArgsFileExists(Path argspath, Selection mode) { + if (!Files.exists(argsPath)) { + switch (mode) { + case ErrorIfMissing: + throw new RuntimeException("A required argsfile was specified, but it does not exist: '" + argspath + "'"); + case WarnIfMissing: + logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'"); + case IgnoreIfMissing: + } + return false; + } + return true; + } + + private void setArgsFile(String argspath, Selection mode) { + Path selected = null; + String[] possibles = argspath.split(":"); + for (String possible : possibles) { + Optional expanded = Environment.INSTANCE.interpolate(possible); + if (expanded.isPresent()) { + Path possiblePath = Path.of(expanded.get()); + if (Files.exists(possiblePath)) { + selected = possiblePath; + break; + } + } + } + + if (selected == null) { + String defaultFirst = possibles[0]; + defaultFirst = Environment.INSTANCE.interpolate(defaultFirst) + .orElseThrow(() -> new RuntimeException("Invalid default argsfile: '" + possibles[0] + "'")); + selected = Path.of(defaultFirst); + } + + this.argsPath = selected; + logger.debug("argsfile path is now '" + this.argsPath.toString() + "'"); + } + + /** + * Convert argv arguments to consolidated form which is used in the args file. + * This means that options and their (optional) arguments are on the + * same line, concatenated with a space after the option. + * + * @return The arg-per-line form + */ + LinkedHashSet argsToLines(List args) { + LinkedHashSet lines = new LinkedHashSet<>(); + Iterator iter = args.iterator(); + List element = new ArrayList<>(); + while (iter.hasNext()) { + String word = iter.next(); + if (word.startsWith("-")) { + if (element.size() > 0) { + lines.add(Strings.join(element, " ")); + element.clear(); + } + } + element.add(word); + } + lines.add(Strings.join(element, " ")); + return lines; + } + + /** + * Convert arg lines as used in an args file to the argv which + * is used on the command line. + * + * @return The argv list as you would see with {@code main(String[] argv)} + */ + LinkedList linesToArgs(Collection lines) { + LinkedList args = new LinkedList<>(); + for (String line : lines) { + if (line.startsWith("-")) { + String[] words = line.split(" ", 2); + args.addAll(Arrays.asList(words)); + } else { + args.add(line); + } + } + return args; + } + + private LinkedList unpin(LinkedList arglist) { + if (this.argsPath == null) { + throw new RuntimeException("No argsfile has been selected before using the pin option."); + } + return arglist; + } + + /** + * Read the current command line option from the argument list, + * so long as it is a dash or double-dash option, and is not a + * reserved word, and any argument that goes with it, if any. + * + * @param arglist The command line containing the option + * @return A list containing the current command line option + */ + private LinkedList readOptionAndArg(LinkedList arglist, boolean consume) { + LinkedList option = new LinkedList<>(); + ListIterator iter = arglist.listIterator(); + + if (!iter.hasNext()) { + throw new RuntimeException("Arguments must follow the --pin option"); + } + String opt = iter.next(); + + if (!opt.startsWith("-") || stopWords.contains(opt)) { + throw new RuntimeException("Arguments following the --pin option must not" + + " be commands like '" + opt + "'"); + } + option.add(opt); + if (consume) { + iter.remove(); + } + + if (iter.hasNext()) { + opt = iter.next(); + if (!stopWords.contains(opt) && !opt.startsWith("-")) { + option.add(opt); + if (consume) { + iter.remove(); + } + } + } + return option; + } + + + /** + * Consume the next word from the beginning of the linked list. If there is + * no word to consume, then throw an error with the description. + * + * @param commandline A list of words + * @param description A description of what the next word value is meant to represent + * @return The next word from the list + */ + private String readWordOrThrow(LinkedList commandline, String description) { + String found = commandline.peekFirst(); + if (found == null) { + throw new RuntimeException("Unable to read argument top option for " + description); + } + return commandline.removeFirst(); + } +} diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index c92aa52f3..fe0398152 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -4,11 +4,15 @@ import ch.qos.logback.classic.Level; import io.nosqlbench.engine.api.metrics.IndicatorMode; import io.nosqlbench.engine.api.util.Unit; import io.nosqlbench.engine.core.script.Scenario; +import io.nosqlbench.nb.api.errors.BasicError; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; import java.security.InvalidParameterException; import java.util.*; import java.util.stream.Collectors; @@ -19,19 +23,16 @@ import java.util.stream.Collectors; */ public class NBCLIOptions { - private final static String userHome = System.getProperty("user.home"); - private final static Path defaultOptFile = Path.of(userHome, ".nosqlbench/options"); - - private final static Logger logger = LoggerFactory.getLogger(NBCLIOptions.class); - - // Options which may contextualize other CLI options or commands. - // These must be parsed first - private static final String ARGS_FILE = "--argsfile"; - private static final String ARGS_FILE_DEFAULT = "$HOME/.nosqlbench/argsfile"; - private static final String ARGS_PIN = "--pin"; - private static final String ARGS_UNPIN = "--unpin"; + private final static Logger logger = LoggerFactory.getLogger("OPTIONS"); + private final static String NB_STATE_DIR = "--statedir"; + private final static String NB_STATEDIR_PATHS = "$NBSTATEDIR:$PWD/.nosqlbench:$HOME/.nosqlbench"; + public static final String ARGS_FILE_DEFAULT = "$NBSTATEDIR/argsfile"; private static final String INCLUDE = "--include"; + + private final static String userHome = System.getProperty("user.home"); + + private static final String METRICS_PREFIX = "--metrics-prefix"; // private static final String ANNOTATE_TO_GRAFANA = "--grafana-baseurl"; @@ -39,7 +40,6 @@ public class NBCLIOptions { private static final String ANNOTATORS_CONFIG = "--annotators"; private static final String DEFAULT_ANNOTATORS = "all"; - // Discovery private static final String HELP = "--help"; private static final String LIST_METRICS = "--list-metrics"; @@ -91,6 +91,7 @@ public class NBCLIOptions { private static final String DOCKER_GRAFANA_TAG = "--docker-grafana-tag"; private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"; + public static final String NBSTATEDIR = "NBSTATEDIR"; private final LinkedList cmdList = new LinkedList<>(); private int logsMax = 0; @@ -130,13 +131,16 @@ public class NBCLIOptions { private Scenario.Engine engine = Scenario.Engine.Graalvm; private boolean graaljs_compat = false; private int hdr_digits = 4; - private String docker_grafana_tag = "7.0.1"; + private String docker_grafana_tag = "7.2.2"; private boolean showStackTraces = false; private boolean compileScript = false; private String scriptFile = null; private String[] annotateEvents = new String[]{"ALL"}; private String dockerMetricsHost; private String annotatorsConfig = ""; + private String statedirs = NB_STATEDIR_PATHS; + private Path statepath; + private List statePathAccesses = new ArrayList<>(); public String getAnnotatorsConfig() { return annotatorsConfig; @@ -163,8 +167,6 @@ public class NBCLIOptions { } private LinkedList parseGlobalOptions(String[] args) { - ArgsFile argsfile = new ArgsFile(); - argsfile.preload("--argsfile-optional", ARGS_FILE_DEFAULT); LinkedList arglist = new LinkedList<>() {{ addAll(Arrays.asList(args)); @@ -175,7 +177,8 @@ public class NBCLIOptions { return arglist; } - // Preprocess --include regardless of position + // Process --include and --statedir, separately first + // regardless of position LinkedList nonincludes = new LinkedList<>(); while (arglist.peekFirst() != null) { String word = arglist.peekFirst(); @@ -188,9 +191,53 @@ public class NBCLIOptions { } switch (word) { - case ARGS_FILE: - case ARGS_PIN: - case ARGS_UNPIN: + case NB_STATE_DIR: + arglist.removeFirst(); + this.statedirs = readWordOrThrow(arglist, "nosqlbench global state directory"); + break; + case INCLUDE: + arglist.removeFirst(); + String include = readWordOrThrow(arglist, "path to include"); + wantsToIncludePaths.add(include); + break; + default: + nonincludes.addLast(arglist.removeFirst()); + } + } + this.statedirs = (this.statedirs != null ? this.statedirs : NB_STATEDIR_PATHS); + this.setStatePath(); + + arglist = nonincludes; + nonincludes = new LinkedList<>(); + + // Now that statdirs is settled, auto load argsfile if it is present + NBCLIArgsFile argsfile = new NBCLIArgsFile(); + argsfile.reserved(NBCLICommandParser.RESERVED_WORDS); + argsfile.preload("--argsfile-optional", ARGS_FILE_DEFAULT); + arglist = argsfile.process(arglist); + + // Parse all --argsfile... and other high level options + + while (arglist.peekFirst() != null) { + String word = arglist.peekFirst(); + if (word.startsWith("--") && word.contains("=")) { + String wordToSplit = arglist.removeFirst(); + String[] split = wordToSplit.split("=", 2); + arglist.offerFirst(split[1]); + arglist.offerFirst(split[0]); + continue; + } + + switch (word) { + // These options modify other options. They should be processed early. + case NBCLIArgsFile.ARGS_FILE: + case NBCLIArgsFile.ARGS_FILE_OPTIONAL: + case NBCLIArgsFile.ARGS_FILE_REQUIRED: + case NBCLIArgsFile.ARGS_PIN: + case NBCLIArgsFile.ARGS_UNPIN: + if (this.statepath == null) { + setStatePath(); + } arglist = argsfile.process(arglist); break; case ANNOTATE_EVENTS: @@ -198,19 +245,10 @@ public class NBCLIOptions { String toAnnotate = readWordOrThrow(arglist, "annotated events"); annotateEvents = toAnnotate.split("\\\\s*,\\\\s*"); break; -// case ANNOTATE_TO_GRAFANA: -// arglist.removeFirst(); -// grafanaEndpoint = readWordOrThrow(arglist,"grafana API endpoint"); -// break; case ANNOTATORS_CONFIG: arglist.removeFirst(); this.annotatorsConfig = readWordOrThrow(arglist, "annotators config"); break; - case INCLUDE: - arglist.removeFirst(); - String include = readWordOrThrow(arglist, "path to include"); - wantsToIncludePaths.add(include); - break; case REPORT_GRAPHITE_TO: arglist.removeFirst(); reportGraphiteTo = arglist.removeFirst(); @@ -245,13 +283,57 @@ public class NBCLIOptions { break; default: nonincludes.addLast(arglist.removeFirst()); - } - } + return nonincludes; } + private Path setStatePath() { + if (statePathAccesses.size() > 0) { + throw new BasicError("The statedir must be set before it is used by other\n" + + " options. If you want to change the statedir, be sure you do it before\n" + + " dependent options. These parameters were called before this --statedir:\n" + + statePathAccesses.stream().map(s -> "> " + s).collect(Collectors.joining("\n"))); + } + if (this.statepath != null) { + return this.statepath; + } + + List paths = Environment.INSTANCE.interpolate(":", statedirs); + Path selected = null; + + for (String pathName : paths) { + Path path = Path.of(pathName); + if (Files.exists(path)) { + if (Files.isDirectory(path)) { + selected = path; + break; + } else { + logger.warn("possible state dir path is not a directory: '" + path.toString() + "'"); + } + } + } + if (selected == null) { + selected = Path.of(paths.get(0)); + } + + if (!Files.exists(selected)) { + try { + Files.createDirectories( + selected, + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwx---")) + ); + } catch (IOException e) { + throw new BasicError("Could not create state directory at '" + selected.toString() + "': " + e.getMessage()); + } + } + + Environment.INSTANCE.put(NBSTATEDIR, selected.toString()); + + return selected; + } + private void parseAllOptions(String[] args) { LinkedList arglist = parseGlobalOptions(args); @@ -565,7 +647,7 @@ public class NBCLIOptions { spec.indicatorMode = IndicatorMode.logonly; } else if (this.getCommands().stream().anyMatch(cmd -> cmd.getCmdType().equals(Cmd.CmdType.script))) { logger.info("Command line includes script calls, so progress data on console is " + - "suppressed."); + "suppressed."); spec.indicatorMode = IndicatorMode.logonly; } } @@ -577,7 +659,7 @@ public class NBCLIOptions { configs.stream().map(LoggerConfig::getFilename).forEach(s -> { if (files.contains(s)) { logger.warn(s + " is included in " + configName + " more than once. It will only be included " + - "in the first matching config. Reorder your options if you need to control this."); + "in the first matching config. Reorder your options if you need to control this."); } files.add(s); }); @@ -688,8 +770,8 @@ public class NBCLIOptions { break; default: throw new RuntimeException( - LOG_HISTOGRAMS + - " options must be in either 'regex:filename:interval' or 'regex:filename' or 'filename' format" + LOG_HISTOGRAMS + + " options must be in either 'regex:filename:interval' or 'regex:filename' or 'filename' format" ); } } @@ -714,7 +796,7 @@ public class NBCLIOptions { switch (parts.length) { case 2: Unit.msFor(parts[1]).orElseThrow( - () -> new RuntimeException("Unable to parse progress indicator indicatorSpec '" + parts[1] + "'") + () -> new RuntimeException("Unable to parse progress indicator indicatorSpec '" + parts[1] + "'") ); progressSpec.intervalSpec = parts[1]; case 1: diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/ArgsFileTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/ArgsFileTest.java deleted file mode 100644 index 91bb05ecc..000000000 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/ArgsFileTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package io.nosqlbench.engine.cli; - -import org.junit.Test; - -import java.util.LinkedList; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ArgsFileTest { - - @Test - public void testLoadingArgs() { - LinkedList result; - ArgsFile argsFile = new ArgsFile(); - result = argsFile.process("--argsfile", "src/test/resources/argsfiles/nonextant.cli"); - assertThat(result).containsExactly(); - result = argsFile.process("--argsfile", "src/test/resources/argsfiles/alphagamma.cli"); - assertThat(result).containsExactly("alpha", "gamma"); - } - - @Test(expected = RuntimeException.class) - public void testLoadingMissingRequiredFails() { - LinkedList result; - ArgsFile argsFile = new ArgsFile(); - result = argsFile.process("--argsfile-required", "src/test/resources/argsfiles/nonextant.cli"); - } - - @Test - public void testLoadingInPlace() { - LinkedList result; - LinkedList commands = new LinkedList<>(List.of("--abc", "--def", "--argsfile", "src/test/resources" + - "/argsfiles/alphagamma.cli")); - ArgsFile argsFile = new ArgsFile().preload("--argsfile-optional", "src/test/resources/argsfiles/alphagamma" + - ".cli"); - result = argsFile.process(commands); - assertThat(result).containsExactly("alpha", "gamma", "--abc", "--def", "alpha", "gamma"); - } - -} \ No newline at end of file diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIArgsFileTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIArgsFileTest.java new file mode 100644 index 000000000..9b2b8f55b --- /dev/null +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIArgsFileTest.java @@ -0,0 +1,117 @@ +package io.nosqlbench.engine.cli; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.LinkedList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NBCLIArgsFileTest { + + @Test + public void testLoadingArgs() { + LinkedList result; + NBCLIArgsFile argsFile = new NBCLIArgsFile(); + result = argsFile.process("--argsfile", "src/test/resources/argsfiles/nonextant.cli"); + assertThat(result).containsExactly(); + result = argsFile.process("--argsfile", "src/test/resources/argsfiles/alphagamma.cli"); + assertThat(result).containsExactly("alpha", "gamma"); + } + + @Test + public void loadParamsWithEnv() { + NBCLIArgsFile argsfile = new NBCLIArgsFile(); + LinkedList result = argsfile.process("--argsfile-required", "src/test/resources/argsfiles/home_env.cli"); + System.out.println(result); + } + + @Test(expected = RuntimeException.class) + public void testLoadingMissingRequiredFails() { + LinkedList result; + NBCLIArgsFile argsFile = new NBCLIArgsFile(); + result = argsFile.process("--argsfile-required", "src/test/resources/argsfiles/nonextant.cli"); + } + + @Test + public void testLoadingInPlace() { + LinkedList result; + LinkedList commands = new LinkedList<>(List.of("--abc", "--def")); + + NBCLIArgsFile argsFile = new NBCLIArgsFile().preload("--argsfile-optional", "src/test/resources/argsfiles/alphagamma" + + ".cli"); + result = argsFile.process(commands); + assertThat(result).containsExactly("alpha", "gamma", "--abc", "--def"); + } + + @Test + public void testLinesToArgs() { + NBCLIArgsFile argsfile = new NBCLIArgsFile().reserved("reservedword"); + LinkedList args = argsfile.linesToArgs( + List.of("--optionname argument", "--optionname2", "--opt3 reservedword") + ); + assertThat(args).containsExactly("--optionname", "argument", "--optionname2", "--opt3", "reservedword"); + } + + @Test + public void testOptionPinning() { + Path tempFile = null; + try { + tempFile = Files.createTempFile( + "tmpfile", + "cli", + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-rw-r--")) + ); + + // preloading is a way to have global defaults based on + // the presence of a default file + NBCLIArgsFile argsfile = new NBCLIArgsFile().preload( + "--argsfile-optional", tempFile.toAbsolutePath().toString() + ); + + LinkedList commandline; + + commandline = argsfile.process( + "--pin", "--option1", + "--pin", "--option1", + "--pin", "--option2", "arg2" + ); + + String filecontents; + filecontents = Files.readString(tempFile); + + // logging should indicate no changes + commandline = argsfile.process( + "--pin", "--option1", + "--pin", "--option1", + "--pin", "--option2", "arg2" + ); + + // unpinned options should be discarded + commandline = argsfile.process( + "--unpin", "--option1", + "--option4", "--option5" + ); + + assertThat(commandline).containsExactly( + "--option4", "--option5" + ); + + assertThat(filecontents).isEqualTo("--option1\n--option2 arg2\n"); + + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + try { + Files.deleteIfExists(tempFile); + } catch (Exception ignored) { + } + } + } + + +} \ No newline at end of file