mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
simplify custom logging configuration
This commit is contained in:
parent
d0592454e8
commit
146686e0b5
@ -10,8 +10,6 @@ import io.nosqlbench.engine.api.metrics.ActivityMetrics;
|
||||
import io.nosqlbench.engine.core.*;
|
||||
import io.nosqlbench.engine.core.annotation.Annotators;
|
||||
import io.nosqlbench.engine.core.logging.LoggerConfig;
|
||||
import io.nosqlbench.engine.core.logging.SessionLogConfig;
|
||||
import io.nosqlbench.engine.core.logging.ScenarioLogger;
|
||||
import io.nosqlbench.engine.core.metrics.MetricReporters;
|
||||
import io.nosqlbench.engine.core.script.MetricsMapper;
|
||||
import io.nosqlbench.engine.core.script.Scenario;
|
||||
@ -45,8 +43,12 @@ import java.util.stream.Collectors;
|
||||
public class NBCLI {
|
||||
|
||||
private static Logger logger;
|
||||
private static LoggerConfig loggerConfig;
|
||||
|
||||
private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
|
||||
static {
|
||||
loggerConfig = new LoggerConfig();
|
||||
LoggerConfig.setConfigurationFactory(loggerConfig);
|
||||
}
|
||||
|
||||
private final String commandName;
|
||||
|
||||
@ -74,13 +76,30 @@ public class NBCLI {
|
||||
// Initial logging config covers only command line parsing
|
||||
// We don't want anything to go to console here unless it is a real problem
|
||||
// as some integrations will depend on a stable and parsable program output
|
||||
LoggerConfig loggerConfig = new LoggerConfig(NBLogLevel.ERROR, NBLogLevel.ERROR);
|
||||
// new LoggerConfig()
|
||||
// .setConsoleLevel(NBLogLevel.INFO.ERROR)
|
||||
// .setLogfileLevel(NBLogLevel.ERROR)
|
||||
// .activate();
|
||||
// logger = LogManager.getLogger("NBCLI");
|
||||
|
||||
ConfigurationFactory.setConfigurationFactory(loggerConfig);
|
||||
logger = LogManager.getLogger("NBCLI");
|
||||
loggerConfig.setConsoleLevel(NBLogLevel.ERROR);
|
||||
|
||||
NBCLIOptions globalOptions = new NBCLIOptions(args, NBCLIOptions.Mode.ParseGlobalsOnly);
|
||||
|
||||
loggerConfig
|
||||
.setConsoleLevel(globalOptions.getConsoleLogLevel())
|
||||
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
|
||||
.setLogfileLevel(globalOptions.getScenarioLogLevel())
|
||||
.getLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
|
||||
.setMaxLogs(globalOptions.getLogsMax())
|
||||
.setLogsDirectory(globalOptions.getLogsDirectory())
|
||||
.activate();
|
||||
ConfigurationFactory.setConfigurationFactory(loggerConfig);
|
||||
|
||||
logger = LogManager.getLogger("NBCLI");
|
||||
loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
|
||||
logger.info("Configured scenario log at " + loggerConfig.getLogfileLocation());
|
||||
|
||||
// Global only processing
|
||||
if (args.length == 0) {
|
||||
System.out.println(loadHelpFile("commandline.md"));
|
||||
@ -139,14 +158,10 @@ public class NBCLI {
|
||||
}
|
||||
|
||||
NBCLIOptions options = new NBCLIOptions(args);
|
||||
NBIO.addGlobalIncludes(options.wantsIncludes());
|
||||
|
||||
String sessionName = new SessionNamer().format(options.getSessionName());
|
||||
logger = LogManager.getLogger("NBCLI");
|
||||
|
||||
SessionLogConfig sessionLogConfig = new SessionLogConfig(sessionName);
|
||||
sessionLogConfig.setConsolePattern(options.getConsoleLoggingPattern());
|
||||
sessionLogConfig.setLevel(options.wantsConsoleLogLevel());
|
||||
sessionLogConfig.start();
|
||||
NBIO.addGlobalIncludes(options.wantsIncludes());
|
||||
|
||||
ActivityMetrics.setHdrDigits(options.getHdrDigits());
|
||||
|
||||
@ -280,27 +295,27 @@ public class NBCLI {
|
||||
if (options.getHistoLoggerConfigs().size() == 0) {
|
||||
logger.info("Adding default histologger configs");
|
||||
String pattern = ".*";
|
||||
String file = CHART_HDR_LOG_NAME;
|
||||
String file = options.getChartHdrFileName();
|
||||
String interval = "1s";
|
||||
options.setHistoLoggerConfigs(pattern, file, interval);
|
||||
}
|
||||
}
|
||||
|
||||
for (
|
||||
NBCLIOptions.LoggerConfig histoLogger : options.getHistoLoggerConfigs()) {
|
||||
NBCLIOptions.LoggerConfigData histoLogger : options.getHistoLoggerConfigs()) {
|
||||
ActivityMetrics.addHistoLogger(sessionName, histoLogger.pattern, histoLogger.file, histoLogger.interval);
|
||||
}
|
||||
for (
|
||||
NBCLIOptions.LoggerConfig statsLogger : options.getStatsLoggerConfigs()) {
|
||||
NBCLIOptions.LoggerConfigData statsLogger : options.getStatsLoggerConfigs()) {
|
||||
ActivityMetrics.addStatsLogger(sessionName, statsLogger.pattern, statsLogger.file, statsLogger.interval);
|
||||
}
|
||||
for (
|
||||
NBCLIOptions.LoggerConfig classicConfigs : options.getClassicHistoConfigs()) {
|
||||
NBCLIOptions.LoggerConfigData classicConfigs : options.getClassicHistoConfigs()) {
|
||||
ActivityMetrics.addClassicHistos(sessionName, classicConfigs.pattern, classicConfigs.file, classicConfigs.interval);
|
||||
}
|
||||
|
||||
// intentionally not shown for warn-only
|
||||
logger.info("console logging level is " + options.wantsConsoleLogLevel());
|
||||
logger.info("console logging level is " + options.getConsoleLogLevel());
|
||||
|
||||
ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1);
|
||||
|
||||
@ -331,13 +346,6 @@ public class NBCLI {
|
||||
}
|
||||
|
||||
|
||||
NBLogLevel consoleLogLevel = options.wantsConsoleLogLevel();
|
||||
NBLogLevel scenarioLogLevel = options.getScenarioLogLevel();
|
||||
if (scenarioLogLevel.isGreaterOrEqualTo(consoleLogLevel)) {
|
||||
logger.info("raising scenario logging level to accommodate console logging level");
|
||||
}
|
||||
NBLogLevel maxLevel = NBLogLevel.max(consoleLogLevel, scenarioLogLevel);
|
||||
|
||||
// Execute Scenario!
|
||||
if (options.getCommands().size() == 0) {
|
||||
logger.info("No commands provided. Exiting before scenario.");
|
||||
@ -349,16 +357,7 @@ public class NBCLI {
|
||||
scriptParams.putAll(buffer.getCombinedParams());
|
||||
scenario.addScenarioScriptParams(scriptParams);
|
||||
|
||||
Path scenarioLogPath = SessionLogConfig.composeSessionLogName(options.getLogsDirectory(), scenario.getScenarioName());
|
||||
logger.info("Configuring scenario log at " + scenarioLogPath.toString());
|
||||
ScenarioLogger sl = new SessionLogConfig(scenario.getScenarioName())
|
||||
.setLogDir(options.getLogsDirectory())
|
||||
.setMaxLogs(options.getLogsMax())
|
||||
.setLevel(maxLevel)
|
||||
.setLogLevelOverrides(options.getLogLevelOverrides())
|
||||
.start();
|
||||
|
||||
executor.execute(scenario, sl);
|
||||
executor.execute(scenario);
|
||||
|
||||
while (true) {
|
||||
Optional<ScenarioResult> pendingResult = executor.getPendingResult(scenario.getScenarioName());
|
||||
|
@ -4,7 +4,6 @@ import io.nosqlbench.nb.api.Environment;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import joptsimple.internal.Strings;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
@ -56,7 +55,8 @@ import java.util.stream.Collectors;
|
||||
* will cause an error to be thrown.
|
||||
*/
|
||||
public class NBCLIArgsFile {
|
||||
private final static Logger logger = LogManager.getLogger("ARGSFILE");
|
||||
private Logger logger;
|
||||
// = LogManager.getLogger("ARGSFILE");
|
||||
|
||||
// Options which may contextualize other CLI options or commands.
|
||||
// These must be parsed first
|
||||
@ -176,10 +176,14 @@ public class NBCLIArgsFile {
|
||||
LinkedHashSet<String> extant = readArgsFile(this.argsPath, Selection.IgnoreIfMissing);
|
||||
LinkedHashSet<String> mergedPins = mergePins(this.argsToPin, this.argsToUnpin, extant);
|
||||
if (extant.equals(mergedPins)) {
|
||||
logger.info("Pinning resulted in no changes to argsfile '" + this.argsPath.toString() + "'");
|
||||
if (logger != null) {
|
||||
logger.info("Pinning resulted in no changes to argsfile '" + this.argsPath.toString() + "'");
|
||||
}
|
||||
} else {
|
||||
logger.info("Writing updated argsfile '" + this.argsPath.toString() + "' with " +
|
||||
(this.argsToPin.size() + this.argsToUnpin.size()) + " changes");
|
||||
if (logger != null) {
|
||||
logger.info("Writing updated argsfile '" + this.argsPath.toString() + "' with " +
|
||||
(this.argsToPin.size() + this.argsToUnpin.size()) + " changes");
|
||||
}
|
||||
writeArgsFile(mergedPins);
|
||||
}
|
||||
|
||||
@ -212,19 +216,27 @@ public class NBCLIArgsFile {
|
||||
|
||||
for (String toAdd : toPin) {
|
||||
if (merged.contains(toAdd)) {
|
||||
logger.warn("Requested to pin argument again: '" + toAdd + "', ignoring");
|
||||
if (logger != null) {
|
||||
logger.warn("Requested to pin argument again: '" + toAdd + "', ignoring");
|
||||
}
|
||||
} else {
|
||||
logger.info("Pinning option '" + toAdd + "' to '" + this.argsPath.toString() + "'");
|
||||
if (logger != null) {
|
||||
logger.info("Pinning option '" + toAdd + "' to '" + this.argsPath.toString() + "'");
|
||||
}
|
||||
merged.add(toAdd);
|
||||
}
|
||||
}
|
||||
|
||||
for (String toDel : toUnpin) {
|
||||
if (merged.contains(toDel)) {
|
||||
logger.info("Unpinning '" + toDel + "' from '" + this.argsPath.toString() + "'");
|
||||
if (logger != null) {
|
||||
logger.info("Unpinning '" + toDel + "' from '" + this.argsPath.toString() + "'");
|
||||
}
|
||||
merged.remove(toDel);
|
||||
} else {
|
||||
logger.warn("Requested to unpin argument '" + toDel + "' which was not found in " + argsPath.toString());
|
||||
if (logger != null) {
|
||||
logger.warn("Requested to unpin argument '" + toDel + "' which was not found in " + argsPath.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -244,7 +256,9 @@ public class NBCLIArgsFile {
|
||||
.map(p -> {
|
||||
String q = Environment.INSTANCE.interpolate(p).orElse(p);
|
||||
if (!q.equals(p)) {
|
||||
logger.info("argsfile: '" + argsPath.toString() + "': loaded option '" + p + "' as '" + q + "'");
|
||||
if (logger != null) {
|
||||
logger.info("argsfile: '" + argsPath.toString() + "': loaded option '" + p + "' as '" + q + "'");
|
||||
}
|
||||
}
|
||||
return q;
|
||||
})
|
||||
@ -360,7 +374,9 @@ public class NBCLIArgsFile {
|
||||
case ErrorIfMissing:
|
||||
throw new RuntimeException("A required argsfile was specified, but it does not exist: '" + argspath + "'");
|
||||
case WarnIfMissing:
|
||||
logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'");
|
||||
if (logger != null) {
|
||||
logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'");
|
||||
}
|
||||
case IgnoreIfMissing:
|
||||
}
|
||||
return false;
|
||||
@ -390,7 +406,9 @@ public class NBCLIArgsFile {
|
||||
}
|
||||
|
||||
this.argsPath = selected;
|
||||
logger.debug("argsfile path is now '" + this.argsPath.toString() + "'");
|
||||
if (logger != null) {
|
||||
logger.debug("argsfile path is now '" + this.argsPath.toString() + "'");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -24,7 +24,8 @@ import java.util.stream.Collectors;
|
||||
*/
|
||||
public class NBCLIOptions {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger("OPTIONS");
|
||||
// private final static Logger logger = LogManager.getLogger("OPTIONS");
|
||||
|
||||
|
||||
private final static String NB_STATE_DIR = "--statedir";
|
||||
private final static String NB_STATEDIR_PATHS = "$NBSTATEDIR:$PWD/.nosqlbench:$HOME/.nosqlbench";
|
||||
@ -41,6 +42,8 @@ public class NBCLIOptions {
|
||||
private static final String ANNOTATORS_CONFIG = "--annotators";
|
||||
private static final String DEFAULT_ANNOTATORS = "all";
|
||||
|
||||
private final static String DEFAULT_CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
|
||||
|
||||
// Discovery
|
||||
private static final String HELP = "--help";
|
||||
private static final String LIST_METRICS = "--list-metrics";
|
||||
@ -144,11 +147,17 @@ public class NBCLIOptions {
|
||||
private String statedirs = NB_STATEDIR_PATHS;
|
||||
private Path statepath;
|
||||
private List<String> statePathAccesses = new ArrayList<>();
|
||||
private String hdrForChartFileName = DEFAULT_CHART_HDR_LOG_NAME;
|
||||
|
||||
public String getAnnotatorsConfig() {
|
||||
return annotatorsConfig;
|
||||
}
|
||||
|
||||
|
||||
public String getChartHdrFileName() {
|
||||
return hdrForChartFileName;
|
||||
}
|
||||
|
||||
public enum Mode {
|
||||
ParseGlobalsOnly,
|
||||
ParseAllOptions
|
||||
@ -243,6 +252,18 @@ public class NBCLIOptions {
|
||||
}
|
||||
arglist = argsfile.process(arglist);
|
||||
break;
|
||||
case DASH_V_INFO:
|
||||
consoleLevel = NBLogLevel.INFO;
|
||||
arglist.removeFirst();
|
||||
break;
|
||||
case DASH_VV_DEBUG:
|
||||
consoleLevel = NBLogLevel.DEBUG;
|
||||
arglist.removeFirst();
|
||||
break;
|
||||
case DASH_VVV_TRACE:
|
||||
consoleLevel = NBLogLevel.TRACE;
|
||||
arglist.removeFirst();
|
||||
break;
|
||||
case ANNOTATE_EVENTS:
|
||||
arglist.removeFirst();
|
||||
String toAnnotate = readWordOrThrow(arglist, "annotated events");
|
||||
@ -284,6 +305,31 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
dockerMetrics = true;
|
||||
break;
|
||||
case SESSION_NAME:
|
||||
arglist.removeFirst();
|
||||
sessionName = readWordOrThrow(arglist, "a session name");
|
||||
break;
|
||||
case LOGS_DIR:
|
||||
arglist.removeFirst();
|
||||
logsDirectory = readWordOrThrow(arglist, "a log directory");
|
||||
break;
|
||||
case LOGS_MAX:
|
||||
arglist.removeFirst();
|
||||
logsMax = Integer.parseInt(readWordOrThrow(arglist, "max logfiles to keep"));
|
||||
break;
|
||||
case LOGS_LEVEL:
|
||||
arglist.removeFirst();
|
||||
String loglevel = readWordOrThrow(arglist, "a log level");
|
||||
this.logsLevel = NBLogLevel.valueOfName(loglevel);
|
||||
break;
|
||||
case LOG_LEVEL_OVERRIDE:
|
||||
arglist.removeFirst();
|
||||
logLevelsOverrides = parseLogLevelOverrides(readWordOrThrow(arglist, "log levels in name:LEVEL,... format"));
|
||||
break;
|
||||
case WITH_LOGGING_PATTERN:
|
||||
arglist.removeFirst();
|
||||
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
|
||||
break;
|
||||
default:
|
||||
nonincludes.addLast(arglist.removeFirst());
|
||||
}
|
||||
@ -313,7 +359,7 @@ public class NBCLIOptions {
|
||||
selected = path;
|
||||
break;
|
||||
} else {
|
||||
logger.warn("possible state dir path is not a directory: '" + path.toString() + "'");
|
||||
System.err.println("ERROR: possible state dir path is not a directory: '" + path.toString() + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -378,31 +424,10 @@ public class NBCLIOptions {
|
||||
Cmd cmd = Cmd.parseArg(arglist, canonicalizer);
|
||||
wantsMetricsForActivity = cmd.getArg("driver");
|
||||
break;
|
||||
case SESSION_NAME:
|
||||
arglist.removeFirst();
|
||||
sessionName = readWordOrThrow(arglist, "a session name");
|
||||
break;
|
||||
case LOGS_DIR:
|
||||
arglist.removeFirst();
|
||||
logsDirectory = readWordOrThrow(arglist, "a log directory");
|
||||
break;
|
||||
case HDR_DIGITS:
|
||||
arglist.removeFirst();
|
||||
hdr_digits = Integer.parseInt(readWordOrThrow(arglist, "significant digits"));
|
||||
break;
|
||||
case LOGS_MAX:
|
||||
arglist.removeFirst();
|
||||
logsMax = Integer.parseInt(readWordOrThrow(arglist, "max logfiles to keep"));
|
||||
break;
|
||||
case LOGS_LEVEL:
|
||||
arglist.removeFirst();
|
||||
String loglevel = readWordOrThrow(arglist, "a log level");
|
||||
this.logsLevel = NBLogLevel.valueOfName(loglevel);
|
||||
break;
|
||||
case LOG_LEVEL_OVERRIDE:
|
||||
arglist.removeFirst();
|
||||
logLevelsOverrides = parseLogLevelOverrides(readWordOrThrow(arglist, "log levels in name:LEVEL,... format"));
|
||||
break;
|
||||
case PROGRESS:
|
||||
arglist.removeFirst();
|
||||
progressSpec = readWordOrThrow(arglist, "a progress indicator, like 'log:1m' or 'screen:10s', or just 'log' or 'screen'");
|
||||
@ -417,7 +442,6 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
if (arglist.peekFirst() == null) {
|
||||
wantsBasicHelp = true;
|
||||
logger.info("getting basic help");
|
||||
} else {
|
||||
wantsActivityHelp = true;
|
||||
wantsActivityHelpFor = readWordOrThrow(arglist, "topic");
|
||||
@ -467,22 +491,6 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
wantsMarkerTypes = true;
|
||||
break;
|
||||
case DASH_V_INFO:
|
||||
consoleLevel = NBLogLevel.INFO;
|
||||
arglist.removeFirst();
|
||||
break;
|
||||
case DASH_VV_DEBUG:
|
||||
consoleLevel = NBLogLevel.DEBUG;
|
||||
arglist.removeFirst();
|
||||
break;
|
||||
case DASH_VVV_TRACE:
|
||||
consoleLevel = NBLogLevel.TRACE;
|
||||
arglist.removeFirst();
|
||||
break;
|
||||
case WITH_LOGGING_PATTERN:
|
||||
arglist.removeFirst();
|
||||
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
|
||||
break;
|
||||
case LIST_SCENARIOS:
|
||||
arglist.removeFirst();
|
||||
wantsScenariosList = true;
|
||||
@ -532,20 +540,23 @@ public class NBCLIOptions {
|
||||
return graaljs_compat;
|
||||
}
|
||||
|
||||
public List<LoggerConfig> getHistoLoggerConfigs() {
|
||||
List<LoggerConfig> configs = histoLoggerConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
|
||||
public List<LoggerConfigData> getHistoLoggerConfigs() {
|
||||
List<LoggerConfigData> configs =
|
||||
histoLoggerConfigs.stream().map(LoggerConfigData::new).collect(Collectors.toList());
|
||||
checkLoggerConfigs(configs, LOG_HISTOGRAMS);
|
||||
return configs;
|
||||
}
|
||||
|
||||
public List<LoggerConfig> getStatsLoggerConfigs() {
|
||||
List<LoggerConfig> configs = statsLoggerConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
|
||||
public List<LoggerConfigData> getStatsLoggerConfigs() {
|
||||
List<LoggerConfigData> configs =
|
||||
statsLoggerConfigs.stream().map(LoggerConfigData::new).collect(Collectors.toList());
|
||||
checkLoggerConfigs(configs, LOG_HISTOSTATS);
|
||||
return configs;
|
||||
}
|
||||
|
||||
public List<LoggerConfig> getClassicHistoConfigs() {
|
||||
List<LoggerConfig> configs = classicHistoConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
|
||||
public List<LoggerConfigData> getClassicHistoConfigs() {
|
||||
List<LoggerConfigData> configs =
|
||||
classicHistoConfigs.stream().map(LoggerConfigData::new).collect(Collectors.toList());
|
||||
checkLoggerConfigs(configs, CLASSIC_HISTOGRAMS);
|
||||
return configs;
|
||||
}
|
||||
@ -622,7 +633,7 @@ public class NBCLIOptions {
|
||||
return sessionName;
|
||||
}
|
||||
|
||||
public NBLogLevel wantsConsoleLogLevel() {
|
||||
public NBLogLevel getConsoleLogLevel() {
|
||||
return consoleLevel;
|
||||
}
|
||||
|
||||
@ -646,11 +657,12 @@ public class NBCLIOptions {
|
||||
public String getProgressSpec() {
|
||||
ProgressSpec spec = parseProgressSpec(this.progressSpec);// sanity check
|
||||
if (spec.indicatorMode == IndicatorMode.console) {
|
||||
if (NBLogLevel.INFO.isGreaterOrEqualTo(wantsConsoleLogLevel())) {
|
||||
logger.warn("Console is already logging info or more, so progress data on console is suppressed.");
|
||||
if (NBLogLevel.INFO.isGreaterOrEqualTo(getConsoleLogLevel())) {
|
||||
System.err.println("Console is already logging info or more, so progress data on console is " +
|
||||
"suppressed.");
|
||||
spec.indicatorMode = IndicatorMode.logonly;
|
||||
} else if (this.getCommands().stream().anyMatch(cmd -> cmd.getCmdType().equals(Cmd.CmdType.script))) {
|
||||
logger.info("Command line includes script calls, so progress data on console is " +
|
||||
System.err.println("Command line includes script calls, so progress data on console is " +
|
||||
"suppressed.");
|
||||
spec.indicatorMode = IndicatorMode.logonly;
|
||||
}
|
||||
@ -658,11 +670,12 @@ public class NBCLIOptions {
|
||||
return spec.toString();
|
||||
}
|
||||
|
||||
private void checkLoggerConfigs(List<LoggerConfig> configs, String configName) {
|
||||
private void checkLoggerConfigs(List<LoggerConfigData> configs, String configName) {
|
||||
Set<String> files = new HashSet<>();
|
||||
configs.stream().map(LoggerConfig::getFilename).forEach(s -> {
|
||||
configs.stream().map(LoggerConfigData::getFilename).forEach(s -> {
|
||||
if (files.contains(s)) {
|
||||
logger.warn(s + " is included in " + configName + " more than once. It will only be included " +
|
||||
System.err.println(s + " is included in " + configName + " more than once. It will only be " +
|
||||
"included " +
|
||||
"in the first matching config. Reorder your options if you need to control this.");
|
||||
}
|
||||
files.add(s);
|
||||
@ -754,12 +767,12 @@ public class NBCLIOptions {
|
||||
return docker_grafana_tag;
|
||||
}
|
||||
|
||||
public static class LoggerConfig {
|
||||
public static class LoggerConfigData {
|
||||
public String file;
|
||||
public String pattern = ".*";
|
||||
public String interval = "30 seconds";
|
||||
|
||||
public LoggerConfig(String histoLoggerSpec) {
|
||||
public LoggerConfigData(String histoLoggerSpec) {
|
||||
String[] words = histoLoggerSpec.split(":");
|
||||
switch (words.length) {
|
||||
case 3:
|
||||
|
@ -2,136 +2,187 @@ package io.nosqlbench.engine.core.logging;
|
||||
|
||||
import io.nosqlbench.nb.api.logging.NBLogLevel;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.core.Filter;
|
||||
import org.apache.logging.log4j.core.LoggerContext;
|
||||
import org.apache.logging.log4j.core.appender.ConsoleAppender;
|
||||
import org.apache.logging.log4j.core.appender.RollingFileAppender;
|
||||
import org.apache.logging.log4j.core.config.Configuration;
|
||||
import org.apache.logging.log4j.core.config.ConfigurationFactory;
|
||||
import org.apache.logging.log4j.core.config.ConfigurationSource;
|
||||
import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.api.*;
|
||||
import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
//@Plugin(name = "CustomConfigurationFactory", category = ConfigurationFactory.CATEGORY)
|
||||
//@Order(50)
|
||||
// Can't use plugin injection, since we need a tailored instance before logging
|
||||
public class LoggerConfig extends ConfigurationFactory {
|
||||
|
||||
private static final String LOG_PATTERN = "%d{DEFAULT}{GMT} [%t] %-5level: %msg%n";
|
||||
private static final String CONSOLE_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
|
||||
public static final Level ROOT_LOG_LEVEL = Level.ALL;
|
||||
private final NBLogLevel consoleLevel;
|
||||
private final NBLogLevel fileLevel;
|
||||
/**
|
||||
* ArgsFile
|
||||
* Environment
|
||||
* NBCLIOptions
|
||||
*/
|
||||
private static final String DEFAULT_CONSOLE_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
|
||||
private String consolePattern = DEFAULT_CONSOLE_PATTERN;
|
||||
private NBLogLevel consoleLevel = NBLogLevel.DEBUG;
|
||||
|
||||
public LoggerConfig(NBLogLevel consoleLevel, NBLogLevel fileLevel) {
|
||||
this.consoleLevel = consoleLevel;
|
||||
this.fileLevel = fileLevel;
|
||||
private static final String DEFAULT_LOGFILE_PATTERN = "%d{DEFAULT}{GMT} [%t] %-5level: %msg%n";
|
||||
private final String logfilePattern = DEFAULT_LOGFILE_PATTERN;
|
||||
private NBLogLevel fileLevel = NBLogLevel.DEBUG;
|
||||
|
||||
public static final Level ROOT_LOG_LEVEL = Level.ALL;
|
||||
private Map<String, String> logLevelOverrides = new LinkedHashMap<>();
|
||||
private Path loggerDir = Path.of("logs");
|
||||
private String sessionName;
|
||||
private int maxLogfiles = 100;
|
||||
private String logfileLocation;
|
||||
|
||||
|
||||
public LoggerConfig() {
|
||||
}
|
||||
|
||||
public LoggerConfig setConsoleLevel(NBLogLevel level) {
|
||||
this.consoleLevel = level;
|
||||
return this;
|
||||
}
|
||||
|
||||
public LoggerConfig setLogfileLevel(NBLogLevel level) {
|
||||
this.fileLevel = level;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure that what is shown in the logfile includes at a minimum,
|
||||
* everything that is shown on console, but allow it to show more
|
||||
* if configured to do so.
|
||||
*/
|
||||
private NBLogLevel getEffectiveFileLevel() {
|
||||
if (fileLevel.isGreaterOrEqualTo(consoleLevel)) {
|
||||
return fileLevel;
|
||||
} else {
|
||||
return consoleLevel;
|
||||
}
|
||||
}
|
||||
|
||||
public LoggerConfig setMaxLogs(int maxLogfiles) {
|
||||
this.maxLogfiles = maxLogfiles;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
Configuration createConfiguration(final String name, ConfigurationBuilder<BuiltConfiguration> builder) {
|
||||
|
||||
Level internalLoggingStatusThreshold = Level.ERROR;
|
||||
Level builderThresholdLevel = Level.INFO;
|
||||
// Level rootLoggingLevel = Level.INFO;
|
||||
|
||||
RootLoggerComponentBuilder rootBuilder = builder.newRootLogger(ROOT_LOG_LEVEL);
|
||||
|
||||
builder.setConfigurationName(name);
|
||||
|
||||
builder.setStatusLevel(internalLoggingStatusThreshold);
|
||||
|
||||
// builder.add(
|
||||
// builder.newFilter(
|
||||
// "ThresholdFilter",
|
||||
// Filter.Result.ACCEPT,
|
||||
// Filter.Result.NEUTRAL
|
||||
// ).addAttribute("level", builderThresholdLevel)
|
||||
// );
|
||||
builder.add(
|
||||
builder.newFilter(
|
||||
"ThresholdFilter",
|
||||
Filter.Result.ACCEPT,
|
||||
Filter.Result.NEUTRAL
|
||||
).addAttribute("level", builderThresholdLevel)
|
||||
);
|
||||
|
||||
// CONSOLE appender
|
||||
AppenderComponentBuilder appenderBuilder =
|
||||
builder.newAppender("Stdout", "CONSOLE")
|
||||
builder.newAppender("console", "CONSOLE")
|
||||
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
|
||||
|
||||
appenderBuilder.add(builder.newLayout("PatternLayout")
|
||||
.addAttribute("pattern", CONSOLE_PATTERN));
|
||||
.addAttribute("pattern", consolePattern));
|
||||
|
||||
// appenderBuilder.add(
|
||||
// builder.newFilter("MarkerFilter", Filter.Result.DENY, Filter.Result.NEUTRAL)
|
||||
// .addAttribute("marker", "FLOW")
|
||||
// );
|
||||
// .addAttribute("marker", "FLOW"));
|
||||
|
||||
builder.add(appenderBuilder);
|
||||
|
||||
// Log4J internal logging
|
||||
builder.add(builder.newLogger("org.apache.logging.log4j", Level.DEBUG)
|
||||
.add(builder.newAppenderRef("Stdout"))
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.addAttribute("additivity", false));
|
||||
|
||||
if (sessionName != null) {
|
||||
|
||||
// LOGFILE appender
|
||||
if (!Files.exists(loggerDir)) {
|
||||
try {
|
||||
Files.createDirectories(loggerDir);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Unable to create logger directory:" + loggerDir);
|
||||
}
|
||||
}
|
||||
|
||||
LayoutComponentBuilder logfileLayout = builder.newLayout("PatternLayout")
|
||||
.addAttribute("pattern", LOG_PATTERN);
|
||||
// LOGFILE appender
|
||||
LayoutComponentBuilder logfileLayout = builder.newLayout("PatternLayout")
|
||||
.addAttribute("pattern", logfilePattern);
|
||||
|
||||
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
|
||||
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
|
||||
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
|
||||
String filebase = getSessionName().replaceAll("\\s", "_");
|
||||
String logfilePath = loggerDir.resolve(filebase + ".log").toString();
|
||||
this.logfileLocation = logfilePath;
|
||||
String archivePath = loggerDir.resolve(filebase + "-TIMESTAMP.log.gz").toString()
|
||||
.replaceAll("TIMESTAMP", "%d{MM-dd-yy}");
|
||||
|
||||
AppenderComponentBuilder logsAppenderBuilder =
|
||||
builder.newAppender("LOGS_APPENDER", RollingFileAppender.PLUGIN_NAME)
|
||||
.addAttribute("fileName", "logs/irengine.log")
|
||||
.addAttribute("filePattern", "logs/irengine-%d{MM-dd-yy}.log.gz")
|
||||
.addAttribute("append", true)
|
||||
.add(logfileLayout)
|
||||
.addComponent(triggeringPolicy);
|
||||
builder.add(logsAppenderBuilder);
|
||||
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
|
||||
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
|
||||
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
|
||||
|
||||
// LoggerComponentBuilder payloadBuilder =
|
||||
// builder.newLogger(PAYLOADS, Level.TRACE)
|
||||
// .addAttribute("additivity", false);
|
||||
//
|
||||
//
|
||||
// if (consoleConfig.isPaylaodsEnabled()) {
|
||||
// payloadBuilder = payloadBuilder.add(builder.newAppenderRef("Stdout").addAttribute("level",
|
||||
// consoleConfig.getLogLevel()));
|
||||
// }
|
||||
// if (fileConfig.isPaylaodsEnabled()) {
|
||||
// payloadBuilder = payloadBuilder.add(builder.newAppenderRef("LOGS_APPENDER").addAttribute("level",
|
||||
// fileConfig.getLogLevel()));
|
||||
// }
|
||||
// builder.add(payloadBuilder);
|
||||
AppenderComponentBuilder logsAppenderBuilder =
|
||||
builder.newAppender("SCENARIO_APPENDER", RollingFileAppender.PLUGIN_NAME)
|
||||
.addAttribute("fileName", logfilePath)
|
||||
.addAttribute("filePattern", archivePath)
|
||||
.addAttribute("append", false)
|
||||
.add(logfileLayout)
|
||||
.addComponent(triggeringPolicy);
|
||||
builder.add(logsAppenderBuilder);
|
||||
|
||||
rootBuilder.add(
|
||||
builder.newAppenderRef("SCENARIO_APPENDER")
|
||||
.addAttribute("level", Level.valueOf(getEffectiveFileLevel().toString()))
|
||||
);
|
||||
}
|
||||
|
||||
// LoggerComponentBuilder stacktracesBuilder =
|
||||
// builder.newLogger(STACKTRACES, Level.TRACE)
|
||||
// .addAttribute("additivity", false);
|
||||
// if (consoleConfig.isStackTracesEnabled()) {
|
||||
// stacktracesBuilder = payloadBuilder.add(builder.newAppenderRef("Stdout").addAttribute("level",
|
||||
// consoleConfig.getLogLevel()));
|
||||
// }
|
||||
// if (fileConfig.isStackTracesEnabled()) {
|
||||
// stacktracesBuilder = payloadBuilder.add(builder.newAppenderRef("LOGS_APPENDER").addAttribute("level",
|
||||
// fileConfig.getLogLevel()));
|
||||
// }
|
||||
// builder.add(stacktracesBuilder);
|
||||
|
||||
// ROOT logging and appender
|
||||
builder.add(
|
||||
builder.newRootLogger(ROOT_LOG_LEVEL)
|
||||
.add(
|
||||
builder.newAppenderRef("Stdout")
|
||||
.addAttribute("level", Level.valueOf(consoleLevel.toString()))
|
||||
)
|
||||
.add(
|
||||
builder.newAppenderRef("LOGS_APPENDER")
|
||||
.addAttribute("level", Level.valueOf(fileLevel.toString()))
|
||||
rootBuilder.add(
|
||||
builder.newAppenderRef("console")
|
||||
.addAttribute("level",
|
||||
Level.valueOf(consoleLevel.toString())
|
||||
)
|
||||
);
|
||||
|
||||
builder.add(rootBuilder);
|
||||
|
||||
return builder.build();
|
||||
if (logLevelOverrides != null) {
|
||||
logLevelOverrides.forEach((k, v) -> {
|
||||
Level olevel = Level.valueOf(v);
|
||||
builder.add(builder.newLogger(k, olevel)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.add(builder.newAppenderRef("SCENARIO_APPENDER"))
|
||||
.addAttribute("additivity", true));
|
||||
});
|
||||
}
|
||||
|
||||
BuiltConfiguration builtConfig = builder.build();
|
||||
return builtConfig;
|
||||
}
|
||||
|
||||
private String getSessionName() {
|
||||
return sessionName;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -149,4 +200,87 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
protected String[] getSupportedTypes() {
|
||||
return new String[]{"*"};
|
||||
}
|
||||
|
||||
public void activate() {
|
||||
ConfigurationFactory.setConfigurationFactory(this);
|
||||
}
|
||||
|
||||
public LoggerConfig setConsolePattern(String consoleLoggingPattern) {
|
||||
this.consolePattern = consoleLoggingPattern;
|
||||
return this;
|
||||
}
|
||||
|
||||
public LoggerConfig getLoggerLevelOverrides(Map<String, String> logLevelOverrides) {
|
||||
this.logLevelOverrides = logLevelOverrides;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Map<String, String> getLogLevelOverrides() {
|
||||
return logLevelOverrides;
|
||||
}
|
||||
|
||||
public LoggerConfig setSessionName(String sessionName) {
|
||||
this.sessionName = sessionName;
|
||||
return this;
|
||||
}
|
||||
|
||||
public LoggerConfig purgeOldFiles(Logger logger) {
|
||||
if (maxLogfiles == 0) {
|
||||
logger.debug("Not purging old files, since maxLogFiles is 0.");
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
File[] files = loggerDir.toFile().listFiles(new FileFilter() {
|
||||
@Override
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.getPath().endsWith(".log") || pathname.getPath().endsWith(".log.gz");
|
||||
}
|
||||
});
|
||||
|
||||
if (files == null) {
|
||||
return this;
|
||||
}
|
||||
|
||||
List<File> filesList = Arrays.asList(files);
|
||||
int remove = filesList.size() - maxLogfiles;
|
||||
if (remove <= 0) {
|
||||
return this;
|
||||
}
|
||||
|
||||
List<File> toDelete = filesList.stream()
|
||||
.sorted(fileTimeComparator)
|
||||
.limit(remove)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (File file : toDelete) {
|
||||
logger.info("removing extra logfile: " + file.getPath());
|
||||
if (!file.delete()) {
|
||||
logger.warn("unable to delete: " + file);
|
||||
try {
|
||||
Files.delete(file.toPath());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
private static final Comparator<File> fileTimeComparator = new Comparator<File>() {
|
||||
@Override
|
||||
public int compare(File o1, File o2) {
|
||||
return Long.compare(o1.lastModified(), o2.lastModified());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
public String getLogfileLocation() {
|
||||
return logfileLocation;
|
||||
}
|
||||
|
||||
public LoggerConfig setLogsDirectory(Path logsDirectory) {
|
||||
this.loggerDir = logsDirectory;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
@ -1,27 +0,0 @@
|
||||
package io.nosqlbench.engine.core.logging;
|
||||
|
||||
import io.nosqlbench.nb.api.logging.NBLogLevel;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
|
||||
public interface ScenarioLogger {
|
||||
|
||||
ScenarioLogger setLogDir(Path logDir);
|
||||
|
||||
Path getLogDir();
|
||||
|
||||
ScenarioLogger setMaxLogs(int maxLogfiles);
|
||||
|
||||
void purgeOldFiles(Logger logger);
|
||||
|
||||
ScenarioLogger setLevel(NBLogLevel levelname);
|
||||
|
||||
NBLogLevel getLevel();
|
||||
|
||||
ScenarioLogger start();
|
||||
|
||||
ScenarioLogger setLogLevelOverrides(Map<String, String> logLevelOverrides);
|
||||
|
||||
}
|
@ -1,204 +0,0 @@
|
||||
package io.nosqlbench.engine.core.logging;
|
||||
|
||||
import io.nosqlbench.nb.api.logging.NBLogLevel;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.core.appender.ConsoleAppender;
|
||||
import org.apache.logging.log4j.core.config.Configurator;
|
||||
import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory;
|
||||
import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
|
||||
import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
|
||||
import org.apache.logging.log4j.spi.LoggerContext;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class SessionLogConfig implements ScenarioLogger {
|
||||
|
||||
// private static final String LOG_PATTERN = "%d{DEFAULT}{GMT} [%t] %-5level: %msg%n";
|
||||
private static final String DEFAULT_CONSOLE_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
|
||||
private final String session;
|
||||
private String consolePattern = DEFAULT_CONSOLE_PATTERN;
|
||||
// private final Scenario scenario;
|
||||
private Path loggerDir = Path.of("logs");
|
||||
private NBLogLevel logLevel = NBLogLevel.INFO;
|
||||
private int maxLogfiles = 100;
|
||||
private Map<String, String> logLevelOverrides = new LinkedHashMap<>();
|
||||
|
||||
public SessionLogConfig(String session) {
|
||||
this.session = session;
|
||||
}
|
||||
|
||||
public ScenarioLogger setConsolePattern(String consolePattern) {
|
||||
this.consolePattern = consolePattern;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getConsolePattern() {
|
||||
return consolePattern;
|
||||
}
|
||||
|
||||
public void configure() {
|
||||
ConfigurationBuilder<BuiltConfiguration> builder = ConfigurationBuilderFactory.newConfigurationBuilder();
|
||||
builder.setStatusLevel(Level.WARN);
|
||||
|
||||
builder.setConfigurationName("ScenarioLogger");
|
||||
|
||||
// Append simple format to stdout
|
||||
|
||||
AppenderComponentBuilder appenderBuilder =
|
||||
builder.newAppender("console", "CONSOLE")
|
||||
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
|
||||
|
||||
appenderBuilder.add(builder.newLayout("PatternLayout")
|
||||
.addAttribute("pattern", consolePattern));
|
||||
|
||||
builder.add(appenderBuilder);
|
||||
|
||||
|
||||
// create a rolling file appender
|
||||
|
||||
Path scenarioLog = composeSessionLogName(loggerDir, session);
|
||||
|
||||
LayoutComponentBuilder layoutBuilder = builder.newLayout("PatternLayout")
|
||||
// .addAttribute("pattern", "%d [%t] %-5level: %msg%n");
|
||||
.addAttribute("pattern", "%date %level [%thread] %logger{10} [%file:%line] %msg%n");
|
||||
|
||||
appenderBuilder = builder.newAppender("scenariolog", "File")
|
||||
.addAttribute("fileName", scenarioLog.getFileName())
|
||||
.addAttribute("append", false)
|
||||
.add(layoutBuilder);
|
||||
builder.add(appenderBuilder);
|
||||
|
||||
// Assemble the scenario logger instance
|
||||
Level level = Level.valueOf(logLevel.toString());
|
||||
builder.add(builder.newRootLogger(level)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.add(builder.newAppenderRef("scenariolog"))
|
||||
.addAttribute("additivity", true));
|
||||
|
||||
logLevelOverrides.forEach((k, v) -> {
|
||||
Level olevel = Level.valueOf(v);
|
||||
builder.add(builder.newLogger(k, olevel)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.add(builder.newAppenderRef("scenariolog"))
|
||||
.addAttribute("additivity", true));
|
||||
});
|
||||
|
||||
LoggerContext ctx = Configurator.initialize(builder.build());
|
||||
}
|
||||
|
||||
public static Path composeSessionLogName(Path loggerDir, String session) {
|
||||
String logfilePath = loggerDir.toString() + File.separator + ".log";
|
||||
Path resolved = loggerDir.resolve(session.replaceAll("\\s", "_") + ".log");
|
||||
return resolved;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScenarioLogger setLogDir(Path logDir) {
|
||||
this.loggerDir = logDir;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Path getLogDir() {
|
||||
return this.loggerDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScenarioLogger setMaxLogs(int maxLogfiles) {
|
||||
this.maxLogfiles = maxLogfiles;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void purgeOldFiles(Logger logger) {
|
||||
if (maxLogfiles == 0) {
|
||||
logger.debug("Not purging old files, since maxLogFiles is 0.");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
File[] files = loggerDir.toFile().listFiles(new FileFilter() {
|
||||
@Override
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.getPath().endsWith(".log");
|
||||
}
|
||||
});
|
||||
if (files == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<File> filesList = Arrays.asList(files);
|
||||
int remove = filesList.size() - maxLogfiles;
|
||||
if (remove <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<File> toDelete = filesList.stream()
|
||||
.sorted(fileTimeComparator)
|
||||
.limit(remove)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (File file : toDelete) {
|
||||
logger.info("removing extra logfile: " + file.getPath());
|
||||
if (!file.delete()) {
|
||||
logger.warn("unable to delete: " + file);
|
||||
try {
|
||||
Files.delete(file.toPath());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final Comparator<File> fileTimeComparator = new Comparator<File>() {
|
||||
@Override
|
||||
public int compare(File o1, File o2) {
|
||||
return Long.compare(o1.lastModified(), o2.lastModified());
|
||||
}
|
||||
};
|
||||
|
||||
@Override
|
||||
public ScenarioLogger setLevel(NBLogLevel levelname) {
|
||||
this.logLevel = levelname;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public NBLogLevel getLevel() {
|
||||
return logLevel;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScenarioLogger start() {
|
||||
if (!Files.exists(loggerDir)) {
|
||||
try {
|
||||
Files.createDirectories(loggerDir);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Unable to create logger directory:" + loggerDir);
|
||||
}
|
||||
}
|
||||
|
||||
configure();
|
||||
org.apache.logging.log4j.Logger logger = LogManager.getLogger("LOGGER");
|
||||
|
||||
purgeOldFiles(logger);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScenarioLogger setLogLevelOverrides(Map<String, String> logLevelOverrides) {
|
||||
this.logLevelOverrides = logLevelOverrides;
|
||||
return this;
|
||||
}
|
||||
}
|
@ -30,7 +30,7 @@ import java.util.ArrayList;
|
||||
*/
|
||||
public class PolyglotMetricRegistryBindings implements ProxyObject, MetricRegistryListener {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(PolyglotMetricRegistryBindings.class);
|
||||
private final static Logger logger = LogManager.getLogger("METRICS");
|
||||
|
||||
private final MetricRegistry registry;
|
||||
MetricMap metrics = new MetricMap("ROOT",null);
|
||||
@ -75,8 +75,8 @@ public class PolyglotMetricRegistryBindings implements ProxyObject, MetricRegist
|
||||
|
||||
@Override
|
||||
public void onGaugeAdded(String name, Gauge<?> gauge) {
|
||||
metrics.add(name,gauge);
|
||||
logger.info("gauge added: " + name +", " + gauge);
|
||||
metrics.add(name, gauge);
|
||||
logger.info("gauge added: " + name);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -24,7 +24,6 @@ import io.nosqlbench.engine.core.PolyglotScenarioController;
|
||||
import io.nosqlbench.engine.core.ScenarioController;
|
||||
import io.nosqlbench.engine.core.ScenarioResult;
|
||||
import io.nosqlbench.engine.core.annotation.Annotators;
|
||||
import io.nosqlbench.engine.core.logging.ScenarioLogger;
|
||||
import io.nosqlbench.engine.core.metrics.NashornMetricRegistryBindings;
|
||||
import io.nosqlbench.engine.core.metrics.PolyglotMetricRegistryBindings;
|
||||
import io.nosqlbench.nb.api.Layer;
|
||||
@ -59,7 +58,7 @@ import java.util.stream.Collectors;
|
||||
|
||||
public class Scenario implements Callable<ScenarioResult> {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger("SCENARIO");
|
||||
private Logger logger = LogManager.getLogger("SCENARIO");
|
||||
|
||||
private State state = State.Scheduled;
|
||||
|
||||
@ -79,7 +78,6 @@ public class Scenario implements Callable<ScenarioResult> {
|
||||
private boolean wantsGraaljsCompatMode;
|
||||
private ScenarioContext scriptEnv;
|
||||
private final String scenarioName;
|
||||
private ScenarioLogger scenarioLogger;
|
||||
private ScriptParams scenarioScriptParams;
|
||||
private String scriptfile;
|
||||
private Engine engine = Engine.Graalvm;
|
||||
@ -110,6 +108,15 @@ public class Scenario implements Callable<ScenarioResult> {
|
||||
this.wantsCompiledScript = wantsCompiledScript;
|
||||
}
|
||||
|
||||
public Scenario setLogger(Logger logger) {
|
||||
this.logger = logger;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Logger getLogger() {
|
||||
return logger;
|
||||
}
|
||||
|
||||
public Scenario(String name, Engine engine) {
|
||||
this.scenarioName = name;
|
||||
this.engine = engine;
|
||||
@ -353,10 +360,6 @@ public class Scenario implements Callable<ScenarioResult> {
|
||||
return "name:'" + this.getScenarioName() + "'";
|
||||
}
|
||||
|
||||
public void setScenarioLogger(ScenarioLogger scenarioLogger) {
|
||||
this.scenarioLogger = scenarioLogger;
|
||||
}
|
||||
|
||||
public void addScenarioScriptParams(ScriptParams scenarioScriptParams) {
|
||||
this.scenarioScriptParams = scenarioScriptParams;
|
||||
}
|
||||
|
@ -21,20 +21,17 @@ import io.nosqlbench.engine.core.IndexedThreadFactory;
|
||||
import io.nosqlbench.engine.core.ScenarioController;
|
||||
import io.nosqlbench.engine.core.ScenarioResult;
|
||||
import io.nosqlbench.engine.core.ScenariosResults;
|
||||
import io.nosqlbench.engine.core.logging.SessionLogConfig;
|
||||
import io.nosqlbench.engine.core.logging.ScenarioLogger;
|
||||
import io.nosqlbench.nb.api.errors.BasicError;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ScenariosExecutor {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(ScenariosExecutor.class);
|
||||
private final Logger logger = LogManager.getLogger("SCENARIOS");
|
||||
private final LinkedHashMap<String, SubmittedScenario> submitted = new LinkedHashMap<>();
|
||||
|
||||
private final ExecutorService executor;
|
||||
@ -54,15 +51,6 @@ public class ScenariosExecutor {
|
||||
}
|
||||
|
||||
public synchronized void execute(Scenario scenario) {
|
||||
ScenarioLogger logs = new SessionLogConfig(scenario.getScenarioName())
|
||||
.setLogDir(Path.of("logs"))
|
||||
.setMaxLogs(0)
|
||||
.start();
|
||||
execute(scenario, logs);
|
||||
}
|
||||
|
||||
public synchronized void execute(Scenario scenario, ScenarioLogger scenarioLogger) {
|
||||
scenario.setScenarioLogger(scenarioLogger);
|
||||
if (submitted.get(scenario.getScenarioName()) != null) {
|
||||
throw new BasicError("Scenario " + scenario.getScenarioName() + " is already defined. Remove it first to reuse the name.");
|
||||
}
|
||||
|
@ -6,8 +6,6 @@ import io.nosqlbench.engine.cli.Cmd;
|
||||
import io.nosqlbench.engine.cli.NBCLICommandParser;
|
||||
import io.nosqlbench.engine.cli.ScriptBuffer;
|
||||
import io.nosqlbench.engine.core.ScenarioResult;
|
||||
import io.nosqlbench.engine.core.logging.SessionLogConfig;
|
||||
import io.nosqlbench.engine.core.logging.ScenarioLogger;
|
||||
import io.nosqlbench.engine.core.script.Scenario;
|
||||
import io.nosqlbench.engine.core.script.ScenariosExecutor;
|
||||
import io.nosqlbench.engine.rest.services.WorkSpace;
|
||||
@ -115,11 +113,7 @@ public class ScenarioExecutorEndpoint implements WebServiceObject {
|
||||
|
||||
scenario.addScriptText(buffer.getParsedScript());
|
||||
|
||||
ScenarioLogger logger = new SessionLogConfig(scenario.getScenarioName())
|
||||
.setLogDir(workspace.getWorkspacePath().resolve("logs"))
|
||||
.start();
|
||||
|
||||
executor.execute(scenario, logger);
|
||||
executor.execute(scenario);
|
||||
|
||||
return Response.created(UriBuilder.fromResource(ScenarioExecutorEndpoint.class).path(
|
||||
"scenario/" + rq.getScenarioName()).build()).entity("started").build();
|
||||
|
@ -36,7 +36,7 @@ import java.util.regex.Pattern;
|
||||
* invalid in this API, except when provided as a default value.
|
||||
*/
|
||||
public class Environment {
|
||||
private final static Logger logger = LogManager.getLogger("ENVIRONMENT");
|
||||
private Logger logger;
|
||||
|
||||
// package private for testing
|
||||
Environment() {
|
||||
@ -66,7 +66,9 @@ public class Environment {
|
||||
}
|
||||
if (references.containsKey(propname)) {
|
||||
if (references.get(propname).equals(value)) {
|
||||
logger.warn("changing already referenced property '" + propname + "' to same value");
|
||||
if (logger != null) {
|
||||
logger.warn("changing already referenced property '" + propname + "' to same value");
|
||||
}
|
||||
} else {
|
||||
throw new BasicError("Changing already referenced property '" + propname + "' from \n" +
|
||||
"'" + references.get(propname) + "' to '" + value + "' is not supported.\n" +
|
||||
@ -108,7 +110,9 @@ public class Environment {
|
||||
}
|
||||
if (envToProp.containsKey(name.toUpperCase())) {
|
||||
String propName = envToProp.get(name.toUpperCase());
|
||||
logger.debug("redirecting env var '" + name + "' to property '" + propName + "'");
|
||||
if (logger != null) {
|
||||
logger.debug("redirecting env var '" + name + "' to property '" + propName + "'");
|
||||
}
|
||||
value = System.getProperty(propName);
|
||||
if (value != null) {
|
||||
return value;
|
||||
@ -173,7 +177,9 @@ public class Environment {
|
||||
}
|
||||
String value = peek(envvar);
|
||||
if (value == null) {
|
||||
logger.debug("no value found for '" + envvar + "', returning Optional.empty() for '" + word + "'");
|
||||
if (logger != null) {
|
||||
logger.debug("no value found for '" + envvar + "', returning Optional.empty() for '" + word + "'");
|
||||
}
|
||||
return Optional.empty();
|
||||
} else {
|
||||
value = reference(envvar, value);
|
||||
|
@ -19,8 +19,6 @@ package io.nosqlbench.engine.core.script;
|
||||
|
||||
import io.nosqlbench.engine.core.ScenarioResult;
|
||||
import io.nosqlbench.engine.core.ScenariosResults;
|
||||
import io.nosqlbench.engine.core.logging.SessionLogConfig;
|
||||
import io.nosqlbench.engine.core.logging.ScenarioLogger;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
import org.assertj.core.data.Offset;
|
||||
import org.junit.BeforeClass;
|
||||
@ -30,7 +28,6 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@ -69,11 +66,7 @@ public class AsyncScriptIntegrationTests {
|
||||
}
|
||||
s.addScriptText(script);
|
||||
// s.addScriptText("load('classpath:scripts/async/" + scriptname + ".js');");
|
||||
ScenarioLogger scenarioLogger = new SessionLogConfig(scenarioName)
|
||||
.setMaxLogs(0)
|
||||
.setLogDir(Path.of("logs/test"))
|
||||
.start();
|
||||
e.execute(s, scenarioLogger);
|
||||
e.execute(s);
|
||||
ScenariosResults scenariosResults = e.awaitAllResults();
|
||||
ScenarioResult scenarioResult = scenariosResults.getOne();
|
||||
scenarioResult.reportToLog();
|
||||
|
@ -19,8 +19,6 @@ package io.nosqlbench.engine.core.script;
|
||||
|
||||
import io.nosqlbench.engine.core.ScenarioResult;
|
||||
import io.nosqlbench.engine.core.ScenariosResults;
|
||||
import io.nosqlbench.engine.core.logging.SessionLogConfig;
|
||||
import io.nosqlbench.engine.core.logging.ScenarioLogger;
|
||||
import org.apache.commons.compress.utils.IOUtils;
|
||||
import org.assertj.core.data.Offset;
|
||||
import org.junit.BeforeClass;
|
||||
@ -29,7 +27,6 @@ import org.junit.Test;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
@ -64,11 +61,7 @@ public class ScriptIntegrationTests {
|
||||
}
|
||||
s.addScriptText(script);
|
||||
// s.addScriptText("load('classpath:scripts/sync/" + scriptname + ".js');");
|
||||
ScenarioLogger scenarioLogger = new SessionLogConfig(scenarioName)
|
||||
.setMaxLogs(0)
|
||||
.setLogDir(Path.of("logs/test"))
|
||||
.start();
|
||||
e.execute(s, scenarioLogger);
|
||||
e.execute(s);
|
||||
ScenariosResults scenariosResults = e.awaitAllResults();
|
||||
ScenarioResult scenarioResult = scenariosResults.getOne();
|
||||
scenarioResult.reportToLog();
|
||||
|
Loading…
Reference in New Issue
Block a user