mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
allow global param preprocessing before app invocation
This commit is contained in:
Jonathan Shook
parent
77a2207a01
commit
4786a22729
@ -1,33 +1,38 @@
|
||||
package io.nosqlbench.engine.cli;
|
||||
|
||||
import ch.qos.logback.classic.Level;
|
||||
import io.nosqlbench.docsys.core.DocServerApp;
|
||||
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
|
||||
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility;
|
||||
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
|
||||
import io.nosqlbench.engine.api.activityapi.input.InputType;
|
||||
import io.nosqlbench.engine.api.activityapi.output.OutputType;
|
||||
import io.nosqlbench.engine.core.*;
|
||||
import io.nosqlbench.engine.core.script.ScriptParams;
|
||||
import io.nosqlbench.nb.api.content.Content;
|
||||
import io.nosqlbench.nb.api.content.NBIO;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import io.nosqlbench.engine.docker.DockerMetricsManager;
|
||||
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
|
||||
import io.nosqlbench.engine.core.*;
|
||||
import io.nosqlbench.engine.core.metrics.MetricReporters;
|
||||
import io.nosqlbench.engine.core.script.MetricsMapper;
|
||||
import io.nosqlbench.engine.core.script.Scenario;
|
||||
import io.nosqlbench.engine.core.script.ScenariosExecutor;
|
||||
import io.nosqlbench.engine.core.script.ScriptParams;
|
||||
import io.nosqlbench.engine.docker.DockerMetricsManager;
|
||||
import io.nosqlbench.nb.api.content.Content;
|
||||
import io.nosqlbench.nb.api.content.NBIO;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import io.nosqlbench.nb.api.markdown.exporter.MarkdownExporter;
|
||||
import io.nosqlbench.virtdata.userlibs.apps.VirtDataMainApp;
|
||||
import io.nosqlbench.docsys.core.DocServerApp;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.locks.LockSupport;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@ -36,7 +41,7 @@ public class NBCLI {
|
||||
private static final Logger logger = LoggerFactory.getLogger(NBCLI.class);
|
||||
private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
|
||||
|
||||
private String commandName;
|
||||
private final String commandName;
|
||||
|
||||
public NBCLI(String commandName) {
|
||||
this.commandName = commandName;
|
||||
@ -58,6 +63,36 @@ public class NBCLI {
|
||||
}
|
||||
|
||||
public void run(String[] args) {
|
||||
|
||||
NBCLIOptions globalOptions = new NBCLIOptions(args,NBCLIOptions.Mode.ParseGlobalsOnly);
|
||||
|
||||
|
||||
// Global only processing
|
||||
|
||||
String reportGraphiteTo = globalOptions.wantsReportGraphiteTo();
|
||||
|
||||
if (globalOptions.wantsDockerMetrics()) {
|
||||
logger.info("Docker metrics is enabled. Docker must be installed for this to work");
|
||||
DockerMetricsManager dmh = new DockerMetricsManager();
|
||||
Map<String, String> dashboardOptions = Map.of(
|
||||
DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag()
|
||||
);
|
||||
dmh.startMetrics(dashboardOptions);
|
||||
|
||||
String warn = "Docker Containers are started, for grafana and prometheus, hit" +
|
||||
" these urls in your browser: http://<host>:3000 and http://<host>:9090";
|
||||
logger.warn(warn);
|
||||
if (reportGraphiteTo != null) {
|
||||
logger.warn(String.format("Docker metrics are enabled (--docker-metrics)" +
|
||||
" but graphite reporting (--report-graphite-to) is set to %s \n" +
|
||||
"usually only one of the two is configured.",
|
||||
reportGraphiteTo));
|
||||
} else {
|
||||
logger.info("Setting graphite reporting to localhost");
|
||||
reportGraphiteTo = "localhost:9109";
|
||||
}
|
||||
}
|
||||
|
||||
if (args.length > 0 && args[0].toLowerCase().equals("virtdata")) {
|
||||
VirtDataMainApp.main(Arrays.copyOfRange(args, 1, args.length));
|
||||
System.exit(0);
|
||||
@ -180,31 +215,6 @@ public class NBCLI {
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
String reportGraphiteTo = options.wantsReportGraphiteTo();
|
||||
|
||||
if (options.wantsDockerMetrics()) {
|
||||
logger.info("Docker metrics is enabled. Docker must be installed for this to work");
|
||||
DockerMetricsManager dmh = new DockerMetricsManager();
|
||||
Map<String, String> dashboardOptions = Map.of(
|
||||
DockerMetricsManager.GRAFANA_TAG, options.getDockerGrafanaTag()
|
||||
);
|
||||
dmh.startMetrics(dashboardOptions);
|
||||
|
||||
String warn = "Docker Containers are started, for grafana and prometheus, hit" +
|
||||
" these urls in your browser: http://<host>:3000 and http://<host>:9090";
|
||||
logger.warn(warn);
|
||||
if (reportGraphiteTo != null) {
|
||||
logger.warn(String.format("Docker metrics are enabled (--docker-metrics)" +
|
||||
" but graphite reporting (--report-graphite-to) is set to %s \n" +
|
||||
"usually only one of the two is configured.",
|
||||
reportGraphiteTo));
|
||||
} else {
|
||||
//TODO: is this right?
|
||||
logger.info("Setting graphite reporting to localhost");
|
||||
reportGraphiteTo = "localhost:9109";
|
||||
}
|
||||
}
|
||||
|
||||
if (reportGraphiteTo != null || options.wantsReportCsvTo() != null) {
|
||||
MetricReporters reporters = MetricReporters.getInstance();
|
||||
reporters.addRegistry("workloads", ActivityMetrics.getMetricRegistry());
|
||||
|
||||
@ -119,21 +119,31 @@ public class NBCLIOptions {
|
||||
private boolean compileScript = false;
|
||||
private String scriptFile = null;
|
||||
|
||||
public NBCLIOptions(String[] args) {
|
||||
parse(args);
|
||||
public enum Mode {
|
||||
ParseGlobalsOnly,
|
||||
ParseAllOptions
|
||||
}
|
||||
|
||||
private void parse(String[] args) {
|
||||
public NBCLIOptions(String[] args) {
|
||||
this(args,Mode.ParseAllOptions);
|
||||
}
|
||||
|
||||
public NBCLIOptions(String[] args, Mode mode) {
|
||||
switch (mode) {
|
||||
case ParseGlobalsOnly:
|
||||
parseGlobalOptions(args);
|
||||
break;
|
||||
case ParseAllOptions:
|
||||
parseAllOptions(args);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private LinkedList<String> parseGlobalOptions(String[] args) {
|
||||
LinkedList<String> arglist = new LinkedList<>() {{
|
||||
addAll(Arrays.asList(args));
|
||||
}};
|
||||
|
||||
if (arglist.peekFirst() == null) {
|
||||
wantsBasicHelp = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Preprocess --include regardless of position
|
||||
LinkedList<String> nonincludes = new LinkedList<>();
|
||||
while (arglist.peekFirst() != null) {
|
||||
@ -146,30 +156,61 @@ public class NBCLIOptions {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (INCLUDE.equals(word)) {
|
||||
arglist.removeFirst();
|
||||
String include = readWordOrThrow(arglist, "path to include");
|
||||
wantsToIncludePaths.add(include);
|
||||
} else if (METRICS_PREFIX.equals(word)) {
|
||||
arglist.removeFirst();
|
||||
metricsPrefix = arglist.removeFirst();
|
||||
} else {
|
||||
nonincludes.addLast(arglist.removeFirst());
|
||||
switch (word) {
|
||||
case INCLUDE:
|
||||
arglist.removeFirst();
|
||||
String include = readWordOrThrow(arglist, "path to include");
|
||||
wantsToIncludePaths.add(include);
|
||||
break;
|
||||
case METRICS_PREFIX:
|
||||
arglist.removeFirst();
|
||||
metricsPrefix = arglist.removeFirst();
|
||||
break;
|
||||
case WORKSPACES_DIR:
|
||||
arglist.removeFirst();
|
||||
workspacesDirectory = readWordOrThrow(arglist, "a workspaces directory");
|
||||
break;
|
||||
case DOCKER_GRAFANA_TAG:
|
||||
arglist.removeFirst();
|
||||
docker_grafana_tag = readWordOrThrow(arglist, "grafana docker tag");
|
||||
break;
|
||||
case VERSION:
|
||||
arglist.removeFirst();
|
||||
wantsVersionShort = true;
|
||||
break;
|
||||
case VERSION_COORDS:
|
||||
arglist.removeFirst();
|
||||
wantsVersionCoords = true;
|
||||
break;
|
||||
case DOCKER_METRICS:
|
||||
arglist.removeFirst();
|
||||
dockerMetrics = true;
|
||||
break;
|
||||
default:
|
||||
nonincludes.addLast(arglist.removeFirst());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
arglist = nonincludes;
|
||||
nonincludes = new LinkedList<>();
|
||||
return nonincludes;
|
||||
}
|
||||
|
||||
private void parseAllOptions(String[] args) {
|
||||
LinkedList<String> arglist = parseGlobalOptions(args);
|
||||
|
||||
PathCanonicalizer canonicalizer = new PathCanonicalizer(wantsIncludes());
|
||||
|
||||
if (arglist.peekFirst() == null) {
|
||||
wantsBasicHelp = true;
|
||||
return;
|
||||
}
|
||||
|
||||
LinkedList<String> nonincludes = new LinkedList<>();
|
||||
|
||||
while (arglist.peekFirst() != null) {
|
||||
String word = arglist.peekFirst();
|
||||
|
||||
switch (word) {
|
||||
case DOCKER_GRAFANA_TAG:
|
||||
arglist.removeFirst();
|
||||
docker_grafana_tag = readWordOrThrow(arglist, "grafana docker tag");
|
||||
break;
|
||||
case GRAALJS_COMPAT:
|
||||
graaljs_compat = true;
|
||||
arglist.removeFirst();
|
||||
@ -208,10 +249,6 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
logsDirectory = readWordOrThrow(arglist, "a log directory");
|
||||
break;
|
||||
case WORKSPACES_DIR:
|
||||
arglist.removeFirst();
|
||||
workspacesDirectory = readWordOrThrow(arglist, "a workspaces directory");
|
||||
break;
|
||||
case HDR_DIGITS:
|
||||
arglist.removeFirst();
|
||||
hdr_digits = Integer.parseInt(readWordOrThrow(arglist, "significant digits"));
|
||||
@ -232,22 +269,10 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
progressSpec = readWordOrThrow(arglist, "a progress indicator, like 'log:1m' or 'screen:10s', or just 'log' or 'screen'");
|
||||
break;
|
||||
case VERSION:
|
||||
arglist.removeFirst();
|
||||
wantsVersionShort = true;
|
||||
break;
|
||||
case VERSION_COORDS:
|
||||
arglist.removeFirst();
|
||||
wantsVersionCoords = true;
|
||||
break;
|
||||
case ENABLE_CHART:
|
||||
arglist.removeFirst();
|
||||
enableChart = true;
|
||||
break;
|
||||
case DOCKER_METRICS:
|
||||
arglist.removeFirst();
|
||||
dockerMetrics = true;
|
||||
break;
|
||||
case HELP:
|
||||
case "-h":
|
||||
case "help":
|
||||
@ -345,7 +370,7 @@ public class NBCLIOptions {
|
||||
}
|
||||
}
|
||||
arglist = nonincludes;
|
||||
NBCLICommandParser.parse(arglist,cmdList);
|
||||
NBCLICommandParser.parse(arglist, cmdList);
|
||||
}
|
||||
|
||||
|
||||
@ -488,7 +513,7 @@ public class NBCLIOptions {
|
||||
spec.indicatorMode = IndicatorMode.logonly;
|
||||
} else if (this.getCommands().stream().anyMatch(cmd -> cmd.getCmdType().equals(Cmd.CmdType.script))) {
|
||||
logger.info("Command line includes script calls, so progress data on console is " +
|
||||
"suppressed.");
|
||||
"suppressed.");
|
||||
spec.indicatorMode = IndicatorMode.logonly;
|
||||
}
|
||||
}
|
||||
@ -500,7 +525,7 @@ public class NBCLIOptions {
|
||||
configs.stream().map(LoggerConfig::getFilename).forEach(s -> {
|
||||
if (files.contains(s)) {
|
||||
logger.warn(s + " is included in " + configName + " more than once. It will only be included " +
|
||||
"in the first matching config. Reorder your options if you need to control this.");
|
||||
"in the first matching config. Reorder your options if you need to control this.");
|
||||
}
|
||||
files.add(s);
|
||||
});
|
||||
@ -527,13 +552,13 @@ public class NBCLIOptions {
|
||||
}
|
||||
|
||||
public String getScriptFile() {
|
||||
if (scriptFile==null) {
|
||||
return logsDirectory+File.separator+"_SESSIONNAME_"+".js";
|
||||
if (scriptFile == null) {
|
||||
return logsDirectory + File.separator + "_SESSIONNAME_" + ".js";
|
||||
}
|
||||
|
||||
String expanded = scriptFile;
|
||||
if (!expanded.startsWith(File.separator)) {
|
||||
expanded = getLogsDirectory()+File.separator+expanded;
|
||||
expanded = getLogsDirectory() + File.separator + expanded;
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
@ -611,8 +636,8 @@ public class NBCLIOptions {
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException(
|
||||
LOG_HISTOGRAMS +
|
||||
" options must be in either 'regex:filename:interval' or 'regex:filename' or 'filename' format"
|
||||
LOG_HISTOGRAMS +
|
||||
" options must be in either 'regex:filename:interval' or 'regex:filename' or 'filename' format"
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -637,7 +662,7 @@ public class NBCLIOptions {
|
||||
switch (parts.length) {
|
||||
case 2:
|
||||
Unit.msFor(parts[1]).orElseThrow(
|
||||
() -> new RuntimeException("Unable to parse progress indicator indicatorSpec '" + parts[1] + "'")
|
||||
() -> new RuntimeException("Unable to parse progress indicator indicatorSpec '" + parts[1] + "'")
|
||||
);
|
||||
progressSpec.intervalSpec = parts[1];
|
||||
case 1:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user