IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

more fixes

Browse Source
This commit is contained in:
Jonathan Shook
2023-09-29 18:55:13 -05:00
parent 0a3b827372
commit 90519b9337
13 changed files with 276 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java
View File

@@ -1,157 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.cli;
import io.nosqlbench.adapters.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class BasicScriptBuffer implements ScriptBuffer {
private final static Logger logger = LogManager.getLogger(Cmd.class);
private final List<Cmd> commands = new ArrayList<>();
private final StringBuilder sb = new StringBuilder();
private final Map<String, String> scriptParams = new HashMap<>();
// public BasicScriptBuffer() {
// this.createPath = null;
// }
public BasicScriptBuffer() {
}
@Override
public List<Cmd> getCommands() {
return this.commands;
}
public ScriptBuffer add(Cmd cmd) {
commands.add(cmd);
Map<String, String> params = cmd.getParams();
switch (cmd.getCmdType()) {
case script:
// sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
// sb.append(";\n");
combineGlobalParams(scriptParams, cmd);
String scriptData = loadScript(cmd);
sb.append(scriptData);
break;
case fragment:
// sb.append(Cmd.toJSONParams("params", cmd.getParams(), false));
// sb.append(";\n");
combineGlobalParams(scriptParams, cmd);
sb.append(cmd.getArg("script_fragment"));
if (cmd.getArg("script_fragment").endsWith(";")) {
sb.append("\n");
}
break;
case start: // start activity
case run: // run activity
case await: // await activity
case stop: // stop activity
case forceStop: // force stopping activity
case waitMillis:
sb.append("scenario.").append(cmd).append("\n");
//// // Sanity check that this can parse before using it
//// sb.append("scenario.").append(cmd.toString()).append("(")
//// .append(Cmd.toJSONBlock(cmd.getParams(), false))
//// .append(");\n");
//// break;
// sb.append("scenario.awaitActivity(\"").append(cmd.getArg("alias_name")).append("\");\n");
// break;
// sb.append("scenario.stop(\"").append(cmd.getArg("alias_name")).append("\");\n");
// break;
// long millis_to_wait = Long.parseLong(cmd.getArg("millis_to_wait"));
// sb.append("scenario.waitMillis(").append(millis_to_wait).append(");\n");
break;
}
return this;
}
/**
* Merge the params from the command into the global params map, but ensure that users know
* if they are overwriting values, which could cause difficult to find bugs in their scripts.
*
* @param scriptParams The existing global params map
* @param cmd The command containing the new params to merge in
*/
private void combineGlobalParams(Map<String, String> scriptParams, Cmd cmd) {
for (String newkey : cmd.getParams().keySet()) {
String newvalue = cmd.getParams().get(newkey);
if (scriptParams.containsKey(newkey)) {
logger.warn("command '" + cmd.getCmdType() + "' overwrote param '" + newkey + " as " + newvalue);
}
scriptParams.put(newkey, newvalue);
}
}
@Override
public ScriptBuffer add(Cmd... cmds) {
for (Cmd cmd : cmds) {
add(cmd);
}
return this;
}
@Override
public String getParsedScript() {
String scripttext = sb.toString();
// String appended = "//@ sourceURL="+tocreate.toString()+"\n\n" + scripttext;
return scripttext;
}
@Override
public Map<String, String> getCombinedParams() {
return scriptParams;
}
public static String assemble(NBCLIOptions options) {
ScriptBuffer script = new BasicScriptBuffer();
for (Cmd command : options.getCommands()) {
script.add(command);
}
return script.getParsedScript();
}
public static String loadScript(Cmd cmd) {
String scriptData;
String script_path = cmd.getArg("script_path");
logger.debug(() -> "Looking for " + script_path);
Content<?> one = NBIO.all().searchPrefixes("scripts").pathname(script_path).extensionSet("js").one();
scriptData = one.asString();
StrInterpolator interpolator = new StrInterpolator(cmd.getParams());
scriptData = interpolator.apply(scriptData);
return scriptData;
}
}

223
engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java
View File

@@ -1,223 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.cli;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.function.Function;
/**
* Encapsulate Command parsing and structure for the NoSQLBench command line. Commands always have a name, sometimes
* have a list of positional arguments, and sometimes have a map of named parameters. An example of a command tha thas
* both would look like {@code script test.js p1=v1}
*/
public class Cmd {
private final static Logger logger = LogManager.getLogger(Cmd.class);
public enum CmdType {
run(),
start(),
stop(Arg.of("alias_name")),
forceStop(Arg.of("alias_name")),
script(Arg.of("script_path", s -> s)),
await(Arg.of("alias_name")),
waitMillis(Arg.of("millis_to_wait", Long::parseLong)),
fragment(Arg.ofFreeform("script_fragment")),;
private final Arg<?>[] positional;
CmdType(Arg<?>... positional) {
this.positional = positional;
}
public String[] getPositionalArgNames() {
String[] names = new String[positional.length];
for (int i = 0; i < names.length; i++) {
names[i] = positional[i].name;
}
return names;
}
public static CmdType valueOfAnyCase(String cmdname) {
for (CmdType value : values()) {
if (cmdname.equals(value.toString()) || cmdname.equalsIgnoreCase(value.toString())) {
return value;
}
}
return valueOf(cmdname); // let the normal exception take over in this case
}
public Arg<?>[] getPositionalArgs() {
return positional;
}
}
private static final class Arg<T> {
public final String name;
public final Function<String, T> converter;
public final boolean freeform;
public Arg(String name, Function<String, T> converter, boolean freeform) {
this.name = name;
this.converter = converter;
this.freeform = freeform;
}
public static <T> Arg<T> of(String name, Function<String, T> converter) {
return new Arg<>(name, converter, false);
}
public static Arg<String> of(String name) {
return new Arg<>(name, s -> s, false);
}
public static Arg<String> ofFreeform(String name) {
return new Arg<>(name, s -> s, true);
}
}
private final Map<String, String> cmdArgs;
public String getArg(String paramName) {
return this.cmdArgs.get(paramName);
}
private final CmdType cmdType;
public Cmd(CmdType cmdType, Map<String, String> cmdArgs) {
this.cmdArgs = cmdArgs;
this.cmdType = cmdType;
}
public CmdType getCmdType() {
return cmdType;
}
public Map<String, String> getParams() {
return cmdArgs;
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(cmdType.toString());
sb.append("(");
if (getParams().size() > cmdType.positional.length) {
sb.append(toJSONBlock(getParams(), false));
} else {
for (String value : getParams().values()) {
String trimmed = ((value.startsWith("'") && value.endsWith("'"))
|| (value.startsWith("\"")) && value.endsWith("\"")) ?
value.substring(1, value.length() - 1) : value;
sb.append("'").append(trimmed).append("'").append(",");
}
sb.setLength(sb.length() - 1);
}
sb.append(");");
return sb.toString();
}
public static Cmd parseArg(LinkedList<String> arglist, PathCanonicalizer fixer) {
String cmdName = arglist.removeFirst();
CmdType cmdType = CmdType.valueOfAnyCase(cmdName);
Map<String, String> params = new LinkedHashMap<>();
for (Arg<?> arg : cmdType.getPositionalArgs()) {
String nextarg = arglist.peekFirst();
if (nextarg == null) {
throw new InvalidParameterException(
"command '" + cmdName + " requires a value for " + arg.name
+ ", but there were no remaining arguments after it.");
} else if (arg.freeform) {
logger.debug(() -> "freeform parameter:" + nextarg);
} else if (nextarg.contains("=")) {
throw new InvalidParameterException(
"command '" + cmdName + "' requires a value for " + arg.name + "" +
", but a named parameter was found instead: " + nextarg);
} else if (NBCLICommandParser.RESERVED_WORDS.contains(nextarg)) {
throw new InvalidParameterException(
"command '" + cmdName + "' requires a value for " + arg.name
+ ", but a reserved word was found instead: " + nextarg);
}
logger.debug(() -> "cmd name:" + cmdName + ", positional " + arg.name + ": " + nextarg);
params.put(arg.name, arg.converter.apply(arglist.removeFirst()).toString());
}
while (arglist.size() > 0 &&
!NBCLICommandParser.RESERVED_WORDS.contains(arglist.peekFirst())
&& arglist.peekFirst().contains("=")) {
String arg = arglist.removeFirst();
String[] assigned = arg.split("=", 2);
String pname = assigned[0];
String pval = assigned[1];
if (pname.equals("yaml") || pname.equals("workload")) {
pval = fixer.canonicalizePath(pval);
}
if (params.containsKey(pname)) {
throw new InvalidParameterException("parameter '" + pname + "' is already set for '" + cmdType +"' command. For each command," +
" a named parameter may only be set once. Multiple occurrences are disallowed to avoid errors or ambiguity.");
}
params.put(pname, pval);
}
return new Cmd(cmdType, params);
}
public static String toJSONBlock(Map<String, String> map, boolean oneline) {
int klen = map.keySet().stream().mapToInt(String::length).max().orElse(1);
StringBuilder sb = new StringBuilder();
List<String> l = new ArrayList<>();
for (Map.Entry<String, String> entries : map.entrySet()) {
String key = entries.getKey();
String value = sanitizeQuotes(entries.getValue());
if (oneline) {
l.add("'" + key + "':'" + value + "'");
} else {
l.add(" '" + key + "': " + " ".repeat(klen - key.length()) + "'" + value + "'");
}
}
return "{" + (oneline ? "" : "\n") + String.join(",\n", l) + (oneline ? "}" : "\n}");
}
private static String sanitizeQuotes(String value) {
if (value.startsWith("'") && value.endsWith("'")) {
return value.substring(1, value.length() - 1);
}
if (value.startsWith("\"") && value.endsWith("\"")) {
return value.substring(1, value.length() - 1);
}
return value;
}
public static String toJSONParams(String varname, Map<String, String> map, boolean oneline) {
return "// params.size==" + map.size() + "\n" + varname + "=" + toJSONBlock(map, oneline);
}
}

169
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@@ -16,31 +16,30 @@
package io.nosqlbench.engine.cli;
import com.codahale.metrics.Gauge;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.api.annotations.Annotation;
import io.nosqlbench.api.annotations.Layer;
import io.nosqlbench.api.labels.NBLabeledElement;
import io.nosqlbench.api.labels.NBLabels;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import io.nosqlbench.api.engine.metrics.instruments.NBFunctionGauge;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.labels.NBLabeledElement;
import io.nosqlbench.api.labels.NBLabels;
import io.nosqlbench.api.logging.NBLogLevel;
import io.nosqlbench.api.metadata.SessionNamer;
import io.nosqlbench.api.metadata.SystemId;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility;
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
import io.nosqlbench.engine.api.activityapi.input.InputType;
import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.cli.NBCLIOptions.LoggerConfigData;
import io.nosqlbench.engine.cli.NBCLIOptions.Mode;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
import io.nosqlbench.engine.core.clientload.ClientSystemMetricChecker;
import io.nosqlbench.engine.core.clientload.DiskStatsReader;
import io.nosqlbench.engine.core.clientload.LoadAvgReader;
@@ -49,16 +48,12 @@ import io.nosqlbench.engine.core.clientload.NetDevReader;
import io.nosqlbench.engine.core.clientload.StatReader;
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
import io.nosqlbench.engine.core.lifecycle.process.ShutdownManager;
import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosResults;
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
import io.nosqlbench.engine.core.lifecycle.scenario.script.MetricsMapper;
import io.nosqlbench.engine.core.lifecycle.session.NBSession;
import io.nosqlbench.engine.core.logging.LoggerConfig;
import io.nosqlbench.engine.core.metadata.MarkdownFinder;
import io.nosqlbench.engine.core.metrics.MetricReporters;
import io.nosqlbench.engine.core.lifecycle.scenario.script.MetricsMapper;
import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
import io.nosqlbench.engine.core.lifecycle.scenario.script.ScriptParams;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.annotations.ServiceSelector;
import org.apache.logging.log4j.LogManager;
@@ -175,17 +170,17 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
.and(globalOptions.getLabelMap());
NBCLI.loggerConfig
.setSessionName(sessionName)
.setConsoleLevel(globalOptions.getConsoleLogLevel())
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
.setLogfileLevel(globalOptions.getScenarioLogLevel())
.setLogfilePattern(globalOptions.getLogfileLoggingPattern())
.setLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
.setMaxLogs(globalOptions.getLogsMax())
.setLogsDirectory(globalOptions.getLogsDirectory())
.setAnsiEnabled(globalOptions.isEnableAnsi())
.setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger())
.activate();
.setSessionName(sessionName)
.setConsoleLevel(globalOptions.getConsoleLogLevel())
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
.setLogfileLevel(globalOptions.getScenarioLogLevel())
.setLogfilePattern(globalOptions.getLogfileLoggingPattern())
.setLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
.setMaxLogs(globalOptions.getLogsMax())
.setLogsDirectory(globalOptions.getLogsDirectory())
.setAnsiEnabled(globalOptions.isEnableAnsi())
.setDedicatedVerificationLogger(globalOptions.isDedicatedVerificationLogger())
.activate();
ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig);
NBCLI.logger = LogManager.getLogger("NBCLI");
@@ -282,7 +277,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
}
if (options.getWantsListCommands()) {
NBCLICommandParser.RESERVED_WORDS.forEach(System.out::println);
SessionCommandParser.RESERVED_WORDS.forEach(System.out::println);
return NBCLI.EXIT_OK;
}
if (options.wantsActivityTypes()) {
@@ -310,19 +305,19 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
NBCLI.logger.debug(() -> "user requests to copy out " + resourceToCopy);
Optional<Content<?>> tocopy = NBIO.classpath()
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocopy.isEmpty()) tocopy = NBIO.classpath()
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).first();
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).first();
final Content<?> data = tocopy.orElseThrow(
() -> new BasicError(
"Unable to find " + resourceToCopy +
" in classpath to copy out")
() -> new BasicError(
"Unable to find " + resourceToCopy +
" in classpath to copy out")
);
final Path writeTo = Path.of(data.asPath().getFileName().toString());
@@ -360,7 +355,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
if (options.wantsTopicalHelp()) {
final Optional<String> helpDoc = MarkdownFinder.forHelpTopic(options.wantsTopicalHelpFor());
System.out.println(helpDoc.orElseThrow(
() -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
() -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
));
return NBCLI.EXIT_OK;
}
@@ -376,23 +371,26 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
NBCLI.logger.debug("initializing annotators with config:'{}'", annotatorsConfig);
Annotators.init(annotatorsConfig, options.getAnnotateLabelSpec());
Annotators.recordAnnotation(
Annotation.newBuilder()
.element(this)
.now()
.layer(Layer.Session)
.addDetail("cli", String.join("\n", args))
.build()
Annotation.newBuilder()
.element(this)
.now()
.layer(Layer.Session)
.addDetail("cli", String.join("\n", args))
.build()
);
if ((null != reportPromPushTo) || (null != reportGraphiteTo) || (null != options.wantsReportCsvTo())) {
final MetricReporters reporters = MetricReporters.getInstance();
reporters.addRegistry("workloads", ActivityMetrics.getMetricRegistry());
if (null != reportPromPushTo) reporters.addPromPush(reportPromPushTo, options.wantsMetricsPrefix(), promPushConfig);
if (null != reportPromPushTo)
reporters.addPromPush(reportPromPushTo, options.wantsMetricsPrefix(), promPushConfig);
if (null != reportGraphiteTo) reporters.addGraphite(reportGraphiteTo, options.wantsMetricsPrefix());
if (null != options.wantsReportCsvTo())
reporters.addCSVReporter(options.wantsReportCsvTo(), options.wantsMetricsPrefix());
if (options.wantsLoggedMetrics()) { reporters.addLogger(); }
if (options.wantsLoggedMetrics()) {
reporters.addLogger();
}
reporters.start(10, options.getReportInterval());
}
@@ -408,13 +406,13 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
}
for (
final LoggerConfigData histoLogger : options.getHistoLoggerConfigs())
final LoggerConfigData histoLogger : options.getHistoLoggerConfigs())
ActivityMetrics.addHistoLogger(sessionName, histoLogger.pattern, histoLogger.file, histoLogger.interval);
for (
final LoggerConfigData statsLogger : options.getStatsLoggerConfigs())
final LoggerConfigData statsLogger : options.getStatsLoggerConfigs())
ActivityMetrics.addStatsLogger(sessionName, statsLogger.pattern, statsLogger.file, statsLogger.interval);
for (
final LoggerConfigData classicConfigs : options.getClassicHistoConfigs())
final LoggerConfigData classicConfigs : options.getClassicHistoConfigs())
ActivityMetrics.addClassicHistos(sessionName, classicConfigs.pattern, classicConfigs.file, classicConfigs.interval);
// client machine metrics; TODO: modify pollInterval
@@ -435,70 +433,29 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
NBCLI.logger.debug(() -> "enabling stack traces since log level is " + options.getConsoleLogLevel());
}
final Scenario scenario = new Scenario(
sessionName,
options.getScriptFile(),
options.getScriptingEngine(),
options.getProgressSpec(),
options.wantsStackTraces(),
options.wantsCompileScript(),
options.getReportSummaryTo(),
String.join("\n", args),
options.getLogsDirectory(),
Maturity.Unspecified,
this);
// intentionally not shown for warn-only
NBCLI.logger.info(() -> "console logging level is " + options.getConsoleLogLevel());
final ScriptBuffer buffer = new BasicScriptBuffer()
.add(options.getCommands()
.toArray(new Cmd[0]));
final String scriptData = buffer.getParsedScript();
if (options.wantsShowScript()) {
System.out.println("// Rendered Script");
System.out.println(scriptData);
return NBCLI.EXIT_OK;
}
if (options.wantsEnableChart()) {
NBCLI.logger.info("Charting enabled");
scenario.enableCharting();
} else NBCLI.logger.info("Charting disabled");
// Execute Scenario!
if (0 == options.getCommands().size()) {
NBCLI.logger.info("No commands provided. Exiting before scenario.");
return NBCLI.EXIT_OK;
}
scenario.addScriptText(scriptData);
final ScriptParams scriptParams = new ScriptParams();
scriptParams.putAll(buffer.getCombinedParams());
scenario.addScenarioScriptParams(scriptParams);
scenariosExecutor.execute(scenario);
final ScenariosResults scenariosResults = scenariosExecutor.awaitAllResults();
NBCLI.logger.debug(() -> "Total of " + scenariosResults.getSize() + " result object returned from ScenariosExecutor");
clientMetricChecker.shutdown();
ActivityMetrics.closeMetrics(options.wantsEnableChart());
scenariosResults.reportToLog();
ShutdownManager.shutdown();
NBCLI.logger.info(scenariosResults.getExecutionSummary());
if (scenariosResults.hasError()) {
final Exception exception = scenariosResults.getOne().getException();
NBCLI.logger.warn(scenariosResults.getExecutionSummary());
NBCLIErrorHandler.handle(exception, options.wantsStackTraces());
System.err.println(exception.getMessage()); // TODO: make this consistent with ConsoleLogging sequencing
return NBCLI.EXIT_ERROR;
}
NBCLI.logger.info(scenariosResults.getExecutionSummary());
return NBCLI.EXIT_OK;
/**
* At this point, the command stream from the CLI should be handed into the session, and the session should
* marshal and transform it for any scenario invocations directly.
*/
NBSession session = new NBSession(
this,
sessionName,
options.getProgressSpec(),
options.getReportSummaryTo(),
options.getLogsDirectory(),
options.getScriptFile(),
options.wantsShowScript()
);
ExecutionResult sessionResult = session.apply(options.getCommands());
sessionResult.printSummary(System.out);
return sessionResult.getStatus().code;
}
private String loadHelpFile(final String filename) {
final ClassLoader cl = this.getClass().getClassLoader();
final InputStream resourceAsStream = cl.getResourceAsStream(filename);

97
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLICommandParser.java
View File

@@ -1,97 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.cli;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import java.util.*;
/**
* This parser will return a non-empty optional if there is no error.
* If the optional is empty, then it means some part of the command structure
* was not recognized.
*/
public class NBCLICommandParser {
private static final String FRAGMENT = "fragment";
private static final String SCRIPT = "script";
private static final String START = "start";
private static final String RUN = "run";
private static final String AWAIT = "await";
private static final String STOP = "stop";
private static final String FORCE_STOP = "forceStop";
private static final String ACTIVITY = "activity";
private static final String SCENARIO = "scenario";
private static final String WAIT_MILLIS = "waitmillis";
public static final Set<String> RESERVED_WORDS = new HashSet<>() {{
addAll(
Arrays.asList(
FRAGMENT, SCRIPT, START, RUN, AWAIT, STOP, FORCE_STOP, ACTIVITY, SCENARIO, WAIT_MILLIS
)
);
}};
public static Optional<List<Cmd>> parse(
LinkedList<String> arglist,
String... includes
) {
List<Cmd> cmdList = new LinkedList<>();
PathCanonicalizer canonicalizer = new PathCanonicalizer(includes);
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
Cmd cmd;
switch (word) {
case FRAGMENT:
case SCRIPT:
case START:
case RUN:
case AWAIT:
case STOP:
case FORCE_STOP:
case WAIT_MILLIS:
cmd = Cmd.parseArg(arglist, canonicalizer);
cmdList.add(cmd);
break;
default:
Optional<Content<?>> scriptfile = NBIO.local()
.searchPrefixes("scripts/auto")
.pathname(word)
.extensionSet("js")
.first();
//Script
if (scriptfile.isPresent()) {
arglist.removeFirst();
arglist.addFirst("scripts/auto/" + word);
arglist.addFirst("script");
cmd = Cmd.parseArg(arglist, canonicalizer);
cmdList.add(cmd);
} else if (NBCLIScenarioParser.isFoundWorkload(word, includes)) {
NBCLIScenarioParser.parseScenarioCommand(arglist, RESERVED_WORDS, includes);
} else {
return Optional.empty();
}
break;
}
}
return Optional.of(cmdList);
}
}

9
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@@ -16,15 +16,14 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.api.labels.NBLabelSpec;
import io.nosqlbench.api.labels.NBLabels;
import io.nosqlbench.api.engine.util.Unit;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.labels.NBLabelSpec;
import io.nosqlbench.api.labels.NBLabels;
import io.nosqlbench.api.logging.NBLogLevel;
import io.nosqlbench.api.system.NBStatePath;
import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.cli.Cmd.CmdType;
import io.nosqlbench.engine.core.lifecycle.scenario.Scenario.Engine;
import io.nosqlbench.nb.annotations.Maturity;
import java.io.File;
@@ -323,7 +322,7 @@ public class NBCLIOptions {
// Now that statdirs is settled, auto load argsfile if it is present
final NBCLIArgsFile argsfile = new NBCLIArgsFile();
argsfile.reserved(NBCLICommandParser.RESERVED_WORDS);
argsfile.reserved(SessionCommandParser.RESERVED_WORDS);
argsfile.preload("--argsfile-optional", NBCLIOptions.ARGS_FILE_DEFAULT);
arglist = argsfile.process(arglist);
@@ -645,7 +644,7 @@ public class NBCLIOptions {
}
}
arglist = nonincludes;
final Optional<List<Cmd>> commands = NBCLICommandParser.parse(arglist);
final Optional<List<Cmd>> commands = SessionCommandParser.parse(arglist);
if (commands.isPresent()) cmdList.addAll(commands.get());
else {
final String arg = arglist.peekFirst();

64
engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java
View File

@@ -1,64 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.cli;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.nio.file.FileSystems;
import java.util.Optional;
public class PathCanonicalizer {
private final static Logger logger = LogManager.getLogger(Cmd.class);
private final String[] includes;
public PathCanonicalizer(String... includes) {
this.includes = includes;
}
public String canonicalizePath(String path) {
Optional<Content<?>> found = NBIO.local().searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(path)
.first();
if (found.isPresent()) {
String rewriteTo = found.get().asPath().toString();
String separator = FileSystems.getDefault().getSeparator();
rewriteTo=(rewriteTo.startsWith(separator) ? rewriteTo.substring(1) : rewriteTo);
if (!rewriteTo.equals(path)) {
if (NBIO.local().searchPrefixes("activities").searchPrefixes(includes).pathname(rewriteTo).first().isPresent()) {
logger.info("rewrote path for " + path + " as " + rewriteTo);
return rewriteTo;
} else {
logger.trace(() -> "kept path for " + path + " as " + found.get().asPath().toString());
return path;
}
} else {
logger.trace(() -> "kept path for " + path + " as " + found.get().asPath().toString());
}
} else {
logger.trace(() -> "unable to find " + path + " for path qualification, either it is remote or missing.");
}
return path;
}
}

50
engine-cli/src/main/java/io/nosqlbench/engine/cli/ScriptBuffer.java
View File

@@ -1,50 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.cli;
import java.util.List;
import java.util.Map;
/**
* Add cmd
*/
public interface ScriptBuffer {
List<Cmd> getCommands();
/**
* Add parsed commands to the script buffer
* @param cmd A parsed command
* @return This ScriptBuffer
*/
ScriptBuffer add(Cmd... cmd);
/**
* Get the text image of the combined script with
* all previously added commands included
* @return The script text
*/
String getParsedScript();
/**
* Get a map which contains all of the params which came from
* commands of global scope, like {@code script} and {@code fragment} commands.
* If one of these commands overwrites a named parameter from another,
* an error should be logged at warning or higher level.
* @return A globa params map.
*/
Map<String, String> getCombinedParams();
}