simplify custom logging configuration

This commit is contained in:
Jonathan Shook 2020-11-19 16:31:53 -06:00
parent d0592454e8
commit 146686e0b5
13 changed files with 368 additions and 458 deletions

View File

@ -10,8 +10,6 @@ import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.core.*;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.logging.LoggerConfig;
import io.nosqlbench.engine.core.logging.SessionLogConfig;
import io.nosqlbench.engine.core.logging.ScenarioLogger;
import io.nosqlbench.engine.core.metrics.MetricReporters;
import io.nosqlbench.engine.core.script.MetricsMapper;
import io.nosqlbench.engine.core.script.Scenario;
@ -45,8 +43,12 @@ import java.util.stream.Collectors;
public class NBCLI {
private static Logger logger;
private static LoggerConfig loggerConfig;
private static final String CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
static {
loggerConfig = new LoggerConfig();
LoggerConfig.setConfigurationFactory(loggerConfig);
}
private final String commandName;
@ -74,13 +76,30 @@ public class NBCLI {
// Initial logging config covers only command line parsing
// We don't want anything to go to console here unless it is a real problem
// as some integrations will depend on a stable and parsable program output
LoggerConfig loggerConfig = new LoggerConfig(NBLogLevel.ERROR, NBLogLevel.ERROR);
// new LoggerConfig()
// .setConsoleLevel(NBLogLevel.INFO.ERROR)
// .setLogfileLevel(NBLogLevel.ERROR)
// .activate();
// logger = LogManager.getLogger("NBCLI");
ConfigurationFactory.setConfigurationFactory(loggerConfig);
logger = LogManager.getLogger("NBCLI");
loggerConfig.setConsoleLevel(NBLogLevel.ERROR);
NBCLIOptions globalOptions = new NBCLIOptions(args, NBCLIOptions.Mode.ParseGlobalsOnly);
loggerConfig
.setConsoleLevel(globalOptions.getConsoleLogLevel())
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
.setLogfileLevel(globalOptions.getScenarioLogLevel())
.getLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
.setMaxLogs(globalOptions.getLogsMax())
.setLogsDirectory(globalOptions.getLogsDirectory())
.activate();
ConfigurationFactory.setConfigurationFactory(loggerConfig);
logger = LogManager.getLogger("NBCLI");
loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
logger.info("Configured scenario log at " + loggerConfig.getLogfileLocation());
// Global only processing
if (args.length == 0) {
System.out.println(loadHelpFile("commandline.md"));
@ -139,14 +158,10 @@ public class NBCLI {
}
NBCLIOptions options = new NBCLIOptions(args);
NBIO.addGlobalIncludes(options.wantsIncludes());
String sessionName = new SessionNamer().format(options.getSessionName());
logger = LogManager.getLogger("NBCLI");
SessionLogConfig sessionLogConfig = new SessionLogConfig(sessionName);
sessionLogConfig.setConsolePattern(options.getConsoleLoggingPattern());
sessionLogConfig.setLevel(options.wantsConsoleLogLevel());
sessionLogConfig.start();
NBIO.addGlobalIncludes(options.wantsIncludes());
ActivityMetrics.setHdrDigits(options.getHdrDigits());
@ -280,27 +295,27 @@ public class NBCLI {
if (options.getHistoLoggerConfigs().size() == 0) {
logger.info("Adding default histologger configs");
String pattern = ".*";
String file = CHART_HDR_LOG_NAME;
String file = options.getChartHdrFileName();
String interval = "1s";
options.setHistoLoggerConfigs(pattern, file, interval);
}
}
for (
NBCLIOptions.LoggerConfig histoLogger : options.getHistoLoggerConfigs()) {
NBCLIOptions.LoggerConfigData histoLogger : options.getHistoLoggerConfigs()) {
ActivityMetrics.addHistoLogger(sessionName, histoLogger.pattern, histoLogger.file, histoLogger.interval);
}
for (
NBCLIOptions.LoggerConfig statsLogger : options.getStatsLoggerConfigs()) {
NBCLIOptions.LoggerConfigData statsLogger : options.getStatsLoggerConfigs()) {
ActivityMetrics.addStatsLogger(sessionName, statsLogger.pattern, statsLogger.file, statsLogger.interval);
}
for (
NBCLIOptions.LoggerConfig classicConfigs : options.getClassicHistoConfigs()) {
NBCLIOptions.LoggerConfigData classicConfigs : options.getClassicHistoConfigs()) {
ActivityMetrics.addClassicHistos(sessionName, classicConfigs.pattern, classicConfigs.file, classicConfigs.interval);
}
// intentionally not shown for warn-only
logger.info("console logging level is " + options.wantsConsoleLogLevel());
logger.info("console logging level is " + options.getConsoleLogLevel());
ScenariosExecutor executor = new ScenariosExecutor("executor-" + sessionName, 1);
@ -331,13 +346,6 @@ public class NBCLI {
}
NBLogLevel consoleLogLevel = options.wantsConsoleLogLevel();
NBLogLevel scenarioLogLevel = options.getScenarioLogLevel();
if (scenarioLogLevel.isGreaterOrEqualTo(consoleLogLevel)) {
logger.info("raising scenario logging level to accommodate console logging level");
}
NBLogLevel maxLevel = NBLogLevel.max(consoleLogLevel, scenarioLogLevel);
// Execute Scenario!
if (options.getCommands().size() == 0) {
logger.info("No commands provided. Exiting before scenario.");
@ -349,16 +357,7 @@ public class NBCLI {
scriptParams.putAll(buffer.getCombinedParams());
scenario.addScenarioScriptParams(scriptParams);
Path scenarioLogPath = SessionLogConfig.composeSessionLogName(options.getLogsDirectory(), scenario.getScenarioName());
logger.info("Configuring scenario log at " + scenarioLogPath.toString());
ScenarioLogger sl = new SessionLogConfig(scenario.getScenarioName())
.setLogDir(options.getLogsDirectory())
.setMaxLogs(options.getLogsMax())
.setLevel(maxLevel)
.setLogLevelOverrides(options.getLogLevelOverrides())
.start();
executor.execute(scenario, sl);
executor.execute(scenario);
while (true) {
Optional<ScenarioResult> pendingResult = executor.getPendingResult(scenario.getScenarioName());

View File

@ -4,7 +4,6 @@ import io.nosqlbench.nb.api.Environment;
import io.nosqlbench.nb.api.errors.BasicError;
import joptsimple.internal.Strings;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.io.IOException;
import java.nio.file.Files;
@ -56,7 +55,8 @@ import java.util.stream.Collectors;
* will cause an error to be thrown.
*/
public class NBCLIArgsFile {
private final static Logger logger = LogManager.getLogger("ARGSFILE");
private Logger logger;
// = LogManager.getLogger("ARGSFILE");
// Options which may contextualize other CLI options or commands.
// These must be parsed first
@ -176,10 +176,14 @@ public class NBCLIArgsFile {
LinkedHashSet<String> extant = readArgsFile(this.argsPath, Selection.IgnoreIfMissing);
LinkedHashSet<String> mergedPins = mergePins(this.argsToPin, this.argsToUnpin, extant);
if (extant.equals(mergedPins)) {
logger.info("Pinning resulted in no changes to argsfile '" + this.argsPath.toString() + "'");
if (logger != null) {
logger.info("Pinning resulted in no changes to argsfile '" + this.argsPath.toString() + "'");
}
} else {
logger.info("Writing updated argsfile '" + this.argsPath.toString() + "' with " +
(this.argsToPin.size() + this.argsToUnpin.size()) + " changes");
if (logger != null) {
logger.info("Writing updated argsfile '" + this.argsPath.toString() + "' with " +
(this.argsToPin.size() + this.argsToUnpin.size()) + " changes");
}
writeArgsFile(mergedPins);
}
@ -212,19 +216,27 @@ public class NBCLIArgsFile {
for (String toAdd : toPin) {
if (merged.contains(toAdd)) {
logger.warn("Requested to pin argument again: '" + toAdd + "', ignoring");
if (logger != null) {
logger.warn("Requested to pin argument again: '" + toAdd + "', ignoring");
}
} else {
logger.info("Pinning option '" + toAdd + "' to '" + this.argsPath.toString() + "'");
if (logger != null) {
logger.info("Pinning option '" + toAdd + "' to '" + this.argsPath.toString() + "'");
}
merged.add(toAdd);
}
}
for (String toDel : toUnpin) {
if (merged.contains(toDel)) {
logger.info("Unpinning '" + toDel + "' from '" + this.argsPath.toString() + "'");
if (logger != null) {
logger.info("Unpinning '" + toDel + "' from '" + this.argsPath.toString() + "'");
}
merged.remove(toDel);
} else {
logger.warn("Requested to unpin argument '" + toDel + "' which was not found in " + argsPath.toString());
if (logger != null) {
logger.warn("Requested to unpin argument '" + toDel + "' which was not found in " + argsPath.toString());
}
}
}
@ -244,7 +256,9 @@ public class NBCLIArgsFile {
.map(p -> {
String q = Environment.INSTANCE.interpolate(p).orElse(p);
if (!q.equals(p)) {
logger.info("argsfile: '" + argsPath.toString() + "': loaded option '" + p + "' as '" + q + "'");
if (logger != null) {
logger.info("argsfile: '" + argsPath.toString() + "': loaded option '" + p + "' as '" + q + "'");
}
}
return q;
})
@ -360,7 +374,9 @@ public class NBCLIArgsFile {
case ErrorIfMissing:
throw new RuntimeException("A required argsfile was specified, but it does not exist: '" + argspath + "'");
case WarnIfMissing:
logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'");
if (logger != null) {
logger.warn("An argsfile was specified, but it does not exist: '" + argspath + "'");
}
case IgnoreIfMissing:
}
return false;
@ -390,7 +406,9 @@ public class NBCLIArgsFile {
}
this.argsPath = selected;
logger.debug("argsfile path is now '" + this.argsPath.toString() + "'");
if (logger != null) {
logger.debug("argsfile path is now '" + this.argsPath.toString() + "'");
}
}
/**

View File

@ -24,7 +24,8 @@ import java.util.stream.Collectors;
*/
public class NBCLIOptions {
private final static Logger logger = LogManager.getLogger("OPTIONS");
// private final static Logger logger = LogManager.getLogger("OPTIONS");
private final static String NB_STATE_DIR = "--statedir";
private final static String NB_STATEDIR_PATHS = "$NBSTATEDIR:$PWD/.nosqlbench:$HOME/.nosqlbench";
@ -41,6 +42,8 @@ public class NBCLIOptions {
private static final String ANNOTATORS_CONFIG = "--annotators";
private static final String DEFAULT_ANNOTATORS = "all";
private final static String DEFAULT_CHART_HDR_LOG_NAME = "hdrdata-for-chart.log";
// Discovery
private static final String HELP = "--help";
private static final String LIST_METRICS = "--list-metrics";
@ -144,11 +147,17 @@ public class NBCLIOptions {
private String statedirs = NB_STATEDIR_PATHS;
private Path statepath;
private List<String> statePathAccesses = new ArrayList<>();
private String hdrForChartFileName = DEFAULT_CHART_HDR_LOG_NAME;
public String getAnnotatorsConfig() {
return annotatorsConfig;
}
public String getChartHdrFileName() {
return hdrForChartFileName;
}
public enum Mode {
ParseGlobalsOnly,
ParseAllOptions
@ -243,6 +252,18 @@ public class NBCLIOptions {
}
arglist = argsfile.process(arglist);
break;
case DASH_V_INFO:
consoleLevel = NBLogLevel.INFO;
arglist.removeFirst();
break;
case DASH_VV_DEBUG:
consoleLevel = NBLogLevel.DEBUG;
arglist.removeFirst();
break;
case DASH_VVV_TRACE:
consoleLevel = NBLogLevel.TRACE;
arglist.removeFirst();
break;
case ANNOTATE_EVENTS:
arglist.removeFirst();
String toAnnotate = readWordOrThrow(arglist, "annotated events");
@ -284,6 +305,31 @@ public class NBCLIOptions {
arglist.removeFirst();
dockerMetrics = true;
break;
case SESSION_NAME:
arglist.removeFirst();
sessionName = readWordOrThrow(arglist, "a session name");
break;
case LOGS_DIR:
arglist.removeFirst();
logsDirectory = readWordOrThrow(arglist, "a log directory");
break;
case LOGS_MAX:
arglist.removeFirst();
logsMax = Integer.parseInt(readWordOrThrow(arglist, "max logfiles to keep"));
break;
case LOGS_LEVEL:
arglist.removeFirst();
String loglevel = readWordOrThrow(arglist, "a log level");
this.logsLevel = NBLogLevel.valueOfName(loglevel);
break;
case LOG_LEVEL_OVERRIDE:
arglist.removeFirst();
logLevelsOverrides = parseLogLevelOverrides(readWordOrThrow(arglist, "log levels in name:LEVEL,... format"));
break;
case WITH_LOGGING_PATTERN:
arglist.removeFirst();
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
break;
default:
nonincludes.addLast(arglist.removeFirst());
}
@ -313,7 +359,7 @@ public class NBCLIOptions {
selected = path;
break;
} else {
logger.warn("possible state dir path is not a directory: '" + path.toString() + "'");
System.err.println("ERROR: possible state dir path is not a directory: '" + path.toString() + "'");
}
}
}
@ -378,31 +424,10 @@ public class NBCLIOptions {
Cmd cmd = Cmd.parseArg(arglist, canonicalizer);
wantsMetricsForActivity = cmd.getArg("driver");
break;
case SESSION_NAME:
arglist.removeFirst();
sessionName = readWordOrThrow(arglist, "a session name");
break;
case LOGS_DIR:
arglist.removeFirst();
logsDirectory = readWordOrThrow(arglist, "a log directory");
break;
case HDR_DIGITS:
arglist.removeFirst();
hdr_digits = Integer.parseInt(readWordOrThrow(arglist, "significant digits"));
break;
case LOGS_MAX:
arglist.removeFirst();
logsMax = Integer.parseInt(readWordOrThrow(arglist, "max logfiles to keep"));
break;
case LOGS_LEVEL:
arglist.removeFirst();
String loglevel = readWordOrThrow(arglist, "a log level");
this.logsLevel = NBLogLevel.valueOfName(loglevel);
break;
case LOG_LEVEL_OVERRIDE:
arglist.removeFirst();
logLevelsOverrides = parseLogLevelOverrides(readWordOrThrow(arglist, "log levels in name:LEVEL,... format"));
break;
case PROGRESS:
arglist.removeFirst();
progressSpec = readWordOrThrow(arglist, "a progress indicator, like 'log:1m' or 'screen:10s', or just 'log' or 'screen'");
@ -417,7 +442,6 @@ public class NBCLIOptions {
arglist.removeFirst();
if (arglist.peekFirst() == null) {
wantsBasicHelp = true;
logger.info("getting basic help");
} else {
wantsActivityHelp = true;
wantsActivityHelpFor = readWordOrThrow(arglist, "topic");
@ -467,22 +491,6 @@ public class NBCLIOptions {
arglist.removeFirst();
wantsMarkerTypes = true;
break;
case DASH_V_INFO:
consoleLevel = NBLogLevel.INFO;
arglist.removeFirst();
break;
case DASH_VV_DEBUG:
consoleLevel = NBLogLevel.DEBUG;
arglist.removeFirst();
break;
case DASH_VVV_TRACE:
consoleLevel = NBLogLevel.TRACE;
arglist.removeFirst();
break;
case WITH_LOGGING_PATTERN:
arglist.removeFirst();
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
break;
case LIST_SCENARIOS:
arglist.removeFirst();
wantsScenariosList = true;
@ -532,20 +540,23 @@ public class NBCLIOptions {
return graaljs_compat;
}
public List<LoggerConfig> getHistoLoggerConfigs() {
List<LoggerConfig> configs = histoLoggerConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
public List<LoggerConfigData> getHistoLoggerConfigs() {
List<LoggerConfigData> configs =
histoLoggerConfigs.stream().map(LoggerConfigData::new).collect(Collectors.toList());
checkLoggerConfigs(configs, LOG_HISTOGRAMS);
return configs;
}
public List<LoggerConfig> getStatsLoggerConfigs() {
List<LoggerConfig> configs = statsLoggerConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
public List<LoggerConfigData> getStatsLoggerConfigs() {
List<LoggerConfigData> configs =
statsLoggerConfigs.stream().map(LoggerConfigData::new).collect(Collectors.toList());
checkLoggerConfigs(configs, LOG_HISTOSTATS);
return configs;
}
public List<LoggerConfig> getClassicHistoConfigs() {
List<LoggerConfig> configs = classicHistoConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
public List<LoggerConfigData> getClassicHistoConfigs() {
List<LoggerConfigData> configs =
classicHistoConfigs.stream().map(LoggerConfigData::new).collect(Collectors.toList());
checkLoggerConfigs(configs, CLASSIC_HISTOGRAMS);
return configs;
}
@ -622,7 +633,7 @@ public class NBCLIOptions {
return sessionName;
}
public NBLogLevel wantsConsoleLogLevel() {
public NBLogLevel getConsoleLogLevel() {
return consoleLevel;
}
@ -646,11 +657,12 @@ public class NBCLIOptions {
public String getProgressSpec() {
ProgressSpec spec = parseProgressSpec(this.progressSpec);// sanity check
if (spec.indicatorMode == IndicatorMode.console) {
if (NBLogLevel.INFO.isGreaterOrEqualTo(wantsConsoleLogLevel())) {
logger.warn("Console is already logging info or more, so progress data on console is suppressed.");
if (NBLogLevel.INFO.isGreaterOrEqualTo(getConsoleLogLevel())) {
System.err.println("Console is already logging info or more, so progress data on console is " +
"suppressed.");
spec.indicatorMode = IndicatorMode.logonly;
} else if (this.getCommands().stream().anyMatch(cmd -> cmd.getCmdType().equals(Cmd.CmdType.script))) {
logger.info("Command line includes script calls, so progress data on console is " +
System.err.println("Command line includes script calls, so progress data on console is " +
"suppressed.");
spec.indicatorMode = IndicatorMode.logonly;
}
@ -658,11 +670,12 @@ public class NBCLIOptions {
return spec.toString();
}
private void checkLoggerConfigs(List<LoggerConfig> configs, String configName) {
private void checkLoggerConfigs(List<LoggerConfigData> configs, String configName) {
Set<String> files = new HashSet<>();
configs.stream().map(LoggerConfig::getFilename).forEach(s -> {
configs.stream().map(LoggerConfigData::getFilename).forEach(s -> {
if (files.contains(s)) {
logger.warn(s + " is included in " + configName + " more than once. It will only be included " +
System.err.println(s + " is included in " + configName + " more than once. It will only be " +
"included " +
"in the first matching config. Reorder your options if you need to control this.");
}
files.add(s);
@ -754,12 +767,12 @@ public class NBCLIOptions {
return docker_grafana_tag;
}
public static class LoggerConfig {
public static class LoggerConfigData {
public String file;
public String pattern = ".*";
public String interval = "30 seconds";
public LoggerConfig(String histoLoggerSpec) {
public LoggerConfigData(String histoLoggerSpec) {
String[] words = histoLoggerSpec.split(":");
switch (words.length) {
case 3:

View File

@ -2,136 +2,187 @@ package io.nosqlbench.engine.core.logging;
import io.nosqlbench.nb.api.logging.NBLogLevel;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.Filter;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.ConsoleAppender;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.ConfigurationFactory;
import org.apache.logging.log4j.core.config.ConfigurationSource;
import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder;
import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
import org.apache.logging.log4j.core.config.builder.api.*;
import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;
//@Plugin(name = "CustomConfigurationFactory", category = ConfigurationFactory.CATEGORY)
//@Order(50)
// Can't use plugin injection, since we need a tailored instance before logging
public class LoggerConfig extends ConfigurationFactory {
private static final String LOG_PATTERN = "%d{DEFAULT}{GMT} [%t] %-5level: %msg%n";
private static final String CONSOLE_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
public static final Level ROOT_LOG_LEVEL = Level.ALL;
private final NBLogLevel consoleLevel;
private final NBLogLevel fileLevel;
/**
* ArgsFile
* Environment
* NBCLIOptions
*/
private static final String DEFAULT_CONSOLE_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
private String consolePattern = DEFAULT_CONSOLE_PATTERN;
private NBLogLevel consoleLevel = NBLogLevel.DEBUG;
public LoggerConfig(NBLogLevel consoleLevel, NBLogLevel fileLevel) {
this.consoleLevel = consoleLevel;
this.fileLevel = fileLevel;
private static final String DEFAULT_LOGFILE_PATTERN = "%d{DEFAULT}{GMT} [%t] %-5level: %msg%n";
private final String logfilePattern = DEFAULT_LOGFILE_PATTERN;
private NBLogLevel fileLevel = NBLogLevel.DEBUG;
public static final Level ROOT_LOG_LEVEL = Level.ALL;
private Map<String, String> logLevelOverrides = new LinkedHashMap<>();
private Path loggerDir = Path.of("logs");
private String sessionName;
private int maxLogfiles = 100;
private String logfileLocation;
public LoggerConfig() {
}
public LoggerConfig setConsoleLevel(NBLogLevel level) {
this.consoleLevel = level;
return this;
}
public LoggerConfig setLogfileLevel(NBLogLevel level) {
this.fileLevel = level;
return this;
}
/**
* Ensure that what is shown in the logfile includes at a minimum,
* everything that is shown on console, but allow it to show more
* if configured to do so.
*/
private NBLogLevel getEffectiveFileLevel() {
if (fileLevel.isGreaterOrEqualTo(consoleLevel)) {
return fileLevel;
} else {
return consoleLevel;
}
}
public LoggerConfig setMaxLogs(int maxLogfiles) {
this.maxLogfiles = maxLogfiles;
return this;
}
Configuration createConfiguration(final String name, ConfigurationBuilder<BuiltConfiguration> builder) {
Level internalLoggingStatusThreshold = Level.ERROR;
Level builderThresholdLevel = Level.INFO;
// Level rootLoggingLevel = Level.INFO;
RootLoggerComponentBuilder rootBuilder = builder.newRootLogger(ROOT_LOG_LEVEL);
builder.setConfigurationName(name);
builder.setStatusLevel(internalLoggingStatusThreshold);
// builder.add(
// builder.newFilter(
// "ThresholdFilter",
// Filter.Result.ACCEPT,
// Filter.Result.NEUTRAL
// ).addAttribute("level", builderThresholdLevel)
// );
builder.add(
builder.newFilter(
"ThresholdFilter",
Filter.Result.ACCEPT,
Filter.Result.NEUTRAL
).addAttribute("level", builderThresholdLevel)
);
// CONSOLE appender
AppenderComponentBuilder appenderBuilder =
builder.newAppender("Stdout", "CONSOLE")
builder.newAppender("console", "CONSOLE")
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
appenderBuilder.add(builder.newLayout("PatternLayout")
.addAttribute("pattern", CONSOLE_PATTERN));
.addAttribute("pattern", consolePattern));
// appenderBuilder.add(
// builder.newFilter("MarkerFilter", Filter.Result.DENY, Filter.Result.NEUTRAL)
// .addAttribute("marker", "FLOW")
// );
// .addAttribute("marker", "FLOW"));
builder.add(appenderBuilder);
// Log4J internal logging
builder.add(builder.newLogger("org.apache.logging.log4j", Level.DEBUG)
.add(builder.newAppenderRef("Stdout"))
.add(builder.newAppenderRef("console"))
.addAttribute("additivity", false));
if (sessionName != null) {
// LOGFILE appender
if (!Files.exists(loggerDir)) {
try {
Files.createDirectories(loggerDir);
} catch (Exception e) {
throw new RuntimeException("Unable to create logger directory:" + loggerDir);
}
}
LayoutComponentBuilder logfileLayout = builder.newLayout("PatternLayout")
.addAttribute("pattern", LOG_PATTERN);
// LOGFILE appender
LayoutComponentBuilder logfileLayout = builder.newLayout("PatternLayout")
.addAttribute("pattern", logfilePattern);
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
String filebase = getSessionName().replaceAll("\\s", "_");
String logfilePath = loggerDir.resolve(filebase + ".log").toString();
this.logfileLocation = logfilePath;
String archivePath = loggerDir.resolve(filebase + "-TIMESTAMP.log.gz").toString()
.replaceAll("TIMESTAMP", "%d{MM-dd-yy}");
AppenderComponentBuilder logsAppenderBuilder =
builder.newAppender("LOGS_APPENDER", RollingFileAppender.PLUGIN_NAME)
.addAttribute("fileName", "logs/irengine.log")
.addAttribute("filePattern", "logs/irengine-%d{MM-dd-yy}.log.gz")
.addAttribute("append", true)
.add(logfileLayout)
.addComponent(triggeringPolicy);
builder.add(logsAppenderBuilder);
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
// LoggerComponentBuilder payloadBuilder =
// builder.newLogger(PAYLOADS, Level.TRACE)
// .addAttribute("additivity", false);
//
//
// if (consoleConfig.isPaylaodsEnabled()) {
// payloadBuilder = payloadBuilder.add(builder.newAppenderRef("Stdout").addAttribute("level",
// consoleConfig.getLogLevel()));
// }
// if (fileConfig.isPaylaodsEnabled()) {
// payloadBuilder = payloadBuilder.add(builder.newAppenderRef("LOGS_APPENDER").addAttribute("level",
// fileConfig.getLogLevel()));
// }
// builder.add(payloadBuilder);
AppenderComponentBuilder logsAppenderBuilder =
builder.newAppender("SCENARIO_APPENDER", RollingFileAppender.PLUGIN_NAME)
.addAttribute("fileName", logfilePath)
.addAttribute("filePattern", archivePath)
.addAttribute("append", false)
.add(logfileLayout)
.addComponent(triggeringPolicy);
builder.add(logsAppenderBuilder);
rootBuilder.add(
builder.newAppenderRef("SCENARIO_APPENDER")
.addAttribute("level", Level.valueOf(getEffectiveFileLevel().toString()))
);
}
// LoggerComponentBuilder stacktracesBuilder =
// builder.newLogger(STACKTRACES, Level.TRACE)
// .addAttribute("additivity", false);
// if (consoleConfig.isStackTracesEnabled()) {
// stacktracesBuilder = payloadBuilder.add(builder.newAppenderRef("Stdout").addAttribute("level",
// consoleConfig.getLogLevel()));
// }
// if (fileConfig.isStackTracesEnabled()) {
// stacktracesBuilder = payloadBuilder.add(builder.newAppenderRef("LOGS_APPENDER").addAttribute("level",
// fileConfig.getLogLevel()));
// }
// builder.add(stacktracesBuilder);
// ROOT logging and appender
builder.add(
builder.newRootLogger(ROOT_LOG_LEVEL)
.add(
builder.newAppenderRef("Stdout")
.addAttribute("level", Level.valueOf(consoleLevel.toString()))
)
.add(
builder.newAppenderRef("LOGS_APPENDER")
.addAttribute("level", Level.valueOf(fileLevel.toString()))
rootBuilder.add(
builder.newAppenderRef("console")
.addAttribute("level",
Level.valueOf(consoleLevel.toString())
)
);
builder.add(rootBuilder);
return builder.build();
if (logLevelOverrides != null) {
logLevelOverrides.forEach((k, v) -> {
Level olevel = Level.valueOf(v);
builder.add(builder.newLogger(k, olevel)
.add(builder.newAppenderRef("console"))
.add(builder.newAppenderRef("SCENARIO_APPENDER"))
.addAttribute("additivity", true));
});
}
BuiltConfiguration builtConfig = builder.build();
return builtConfig;
}
private String getSessionName() {
return sessionName;
}
@Override
@ -149,4 +200,87 @@ public class LoggerConfig extends ConfigurationFactory {
protected String[] getSupportedTypes() {
return new String[]{"*"};
}
public void activate() {
ConfigurationFactory.setConfigurationFactory(this);
}
public LoggerConfig setConsolePattern(String consoleLoggingPattern) {
this.consolePattern = consoleLoggingPattern;
return this;
}
public LoggerConfig getLoggerLevelOverrides(Map<String, String> logLevelOverrides) {
this.logLevelOverrides = logLevelOverrides;
return this;
}
public Map<String, String> getLogLevelOverrides() {
return logLevelOverrides;
}
public LoggerConfig setSessionName(String sessionName) {
this.sessionName = sessionName;
return this;
}
public LoggerConfig purgeOldFiles(Logger logger) {
if (maxLogfiles == 0) {
logger.debug("Not purging old files, since maxLogFiles is 0.");
return this;
}
File[] files = loggerDir.toFile().listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.getPath().endsWith(".log") || pathname.getPath().endsWith(".log.gz");
}
});
if (files == null) {
return this;
}
List<File> filesList = Arrays.asList(files);
int remove = filesList.size() - maxLogfiles;
if (remove <= 0) {
return this;
}
List<File> toDelete = filesList.stream()
.sorted(fileTimeComparator)
.limit(remove)
.collect(Collectors.toList());
for (File file : toDelete) {
logger.info("removing extra logfile: " + file.getPath());
if (!file.delete()) {
logger.warn("unable to delete: " + file);
try {
Files.delete(file.toPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
return this;
}
private static final Comparator<File> fileTimeComparator = new Comparator<File>() {
@Override
public int compare(File o1, File o2) {
return Long.compare(o1.lastModified(), o2.lastModified());
}
};
public String getLogfileLocation() {
return logfileLocation;
}
public LoggerConfig setLogsDirectory(Path logsDirectory) {
this.loggerDir = logsDirectory;
return this;
}
}

View File

@ -1,27 +0,0 @@
package io.nosqlbench.engine.core.logging;
import io.nosqlbench.nb.api.logging.NBLogLevel;
import org.apache.logging.log4j.Logger;
import java.nio.file.Path;
import java.util.Map;
public interface ScenarioLogger {
ScenarioLogger setLogDir(Path logDir);
Path getLogDir();
ScenarioLogger setMaxLogs(int maxLogfiles);
void purgeOldFiles(Logger logger);
ScenarioLogger setLevel(NBLogLevel levelname);
NBLogLevel getLevel();
ScenarioLogger start();
ScenarioLogger setLogLevelOverrides(Map<String, String> logLevelOverrides);
}

View File

@ -1,204 +0,0 @@
package io.nosqlbench.engine.core.logging;
import io.nosqlbench.nb.api.logging.NBLogLevel;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.appender.ConsoleAppender;
import org.apache.logging.log4j.core.config.Configurator;
import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory;
import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
import org.apache.logging.log4j.spi.LoggerContext;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;
public class SessionLogConfig implements ScenarioLogger {
// private static final String LOG_PATTERN = "%d{DEFAULT}{GMT} [%t] %-5level: %msg%n";
private static final String DEFAULT_CONSOLE_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
private final String session;
private String consolePattern = DEFAULT_CONSOLE_PATTERN;
// private final Scenario scenario;
private Path loggerDir = Path.of("logs");
private NBLogLevel logLevel = NBLogLevel.INFO;
private int maxLogfiles = 100;
private Map<String, String> logLevelOverrides = new LinkedHashMap<>();
public SessionLogConfig(String session) {
this.session = session;
}
public ScenarioLogger setConsolePattern(String consolePattern) {
this.consolePattern = consolePattern;
return this;
}
public String getConsolePattern() {
return consolePattern;
}
public void configure() {
ConfigurationBuilder<BuiltConfiguration> builder = ConfigurationBuilderFactory.newConfigurationBuilder();
builder.setStatusLevel(Level.WARN);
builder.setConfigurationName("ScenarioLogger");
// Append simple format to stdout
AppenderComponentBuilder appenderBuilder =
builder.newAppender("console", "CONSOLE")
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
appenderBuilder.add(builder.newLayout("PatternLayout")
.addAttribute("pattern", consolePattern));
builder.add(appenderBuilder);
// create a rolling file appender
Path scenarioLog = composeSessionLogName(loggerDir, session);
LayoutComponentBuilder layoutBuilder = builder.newLayout("PatternLayout")
// .addAttribute("pattern", "%d [%t] %-5level: %msg%n");
.addAttribute("pattern", "%date %level [%thread] %logger{10} [%file:%line] %msg%n");
appenderBuilder = builder.newAppender("scenariolog", "File")
.addAttribute("fileName", scenarioLog.getFileName())
.addAttribute("append", false)
.add(layoutBuilder);
builder.add(appenderBuilder);
// Assemble the scenario logger instance
Level level = Level.valueOf(logLevel.toString());
builder.add(builder.newRootLogger(level)
.add(builder.newAppenderRef("console"))
.add(builder.newAppenderRef("scenariolog"))
.addAttribute("additivity", true));
logLevelOverrides.forEach((k, v) -> {
Level olevel = Level.valueOf(v);
builder.add(builder.newLogger(k, olevel)
.add(builder.newAppenderRef("console"))
.add(builder.newAppenderRef("scenariolog"))
.addAttribute("additivity", true));
});
LoggerContext ctx = Configurator.initialize(builder.build());
}
public static Path composeSessionLogName(Path loggerDir, String session) {
String logfilePath = loggerDir.toString() + File.separator + ".log";
Path resolved = loggerDir.resolve(session.replaceAll("\\s", "_") + ".log");
return resolved;
}
@Override
public ScenarioLogger setLogDir(Path logDir) {
this.loggerDir = logDir;
return this;
}
@Override
public Path getLogDir() {
return this.loggerDir;
}
@Override
public ScenarioLogger setMaxLogs(int maxLogfiles) {
this.maxLogfiles = maxLogfiles;
return this;
}
@Override
public void purgeOldFiles(Logger logger) {
if (maxLogfiles == 0) {
logger.debug("Not purging old files, since maxLogFiles is 0.");
return;
}
File[] files = loggerDir.toFile().listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.getPath().endsWith(".log");
}
});
if (files == null) {
return;
}
List<File> filesList = Arrays.asList(files);
int remove = filesList.size() - maxLogfiles;
if (remove <= 0) {
return;
}
List<File> toDelete = filesList.stream()
.sorted(fileTimeComparator)
.limit(remove)
.collect(Collectors.toList());
for (File file : toDelete) {
logger.info("removing extra logfile: " + file.getPath());
if (!file.delete()) {
logger.warn("unable to delete: " + file);
try {
Files.delete(file.toPath());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}
private static final Comparator<File> fileTimeComparator = new Comparator<File>() {
@Override
public int compare(File o1, File o2) {
return Long.compare(o1.lastModified(), o2.lastModified());
}
};
@Override
public ScenarioLogger setLevel(NBLogLevel levelname) {
this.logLevel = levelname;
return this;
}
@Override
public NBLogLevel getLevel() {
return logLevel;
}
@Override
public ScenarioLogger start() {
if (!Files.exists(loggerDir)) {
try {
Files.createDirectories(loggerDir);
} catch (Exception e) {
throw new RuntimeException("Unable to create logger directory:" + loggerDir);
}
}
configure();
org.apache.logging.log4j.Logger logger = LogManager.getLogger("LOGGER");
purgeOldFiles(logger);
return this;
}
@Override
public ScenarioLogger setLogLevelOverrides(Map<String, String> logLevelOverrides) {
this.logLevelOverrides = logLevelOverrides;
return this;
}
}

View File

@ -30,7 +30,7 @@ import java.util.ArrayList;
*/
public class PolyglotMetricRegistryBindings implements ProxyObject, MetricRegistryListener {
private final static Logger logger = LogManager.getLogger(PolyglotMetricRegistryBindings.class);
private final static Logger logger = LogManager.getLogger("METRICS");
private final MetricRegistry registry;
MetricMap metrics = new MetricMap("ROOT",null);
@ -75,8 +75,8 @@ public class PolyglotMetricRegistryBindings implements ProxyObject, MetricRegist
@Override
public void onGaugeAdded(String name, Gauge<?> gauge) {
metrics.add(name,gauge);
logger.info("gauge added: " + name +", " + gauge);
metrics.add(name, gauge);
logger.info("gauge added: " + name);
}
@Override

View File

@ -24,7 +24,6 @@ import io.nosqlbench.engine.core.PolyglotScenarioController;
import io.nosqlbench.engine.core.ScenarioController;
import io.nosqlbench.engine.core.ScenarioResult;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.logging.ScenarioLogger;
import io.nosqlbench.engine.core.metrics.NashornMetricRegistryBindings;
import io.nosqlbench.engine.core.metrics.PolyglotMetricRegistryBindings;
import io.nosqlbench.nb.api.Layer;
@ -59,7 +58,7 @@ import java.util.stream.Collectors;
public class Scenario implements Callable<ScenarioResult> {
private static final Logger logger = LogManager.getLogger("SCENARIO");
private Logger logger = LogManager.getLogger("SCENARIO");
private State state = State.Scheduled;
@ -79,7 +78,6 @@ public class Scenario implements Callable<ScenarioResult> {
private boolean wantsGraaljsCompatMode;
private ScenarioContext scriptEnv;
private final String scenarioName;
private ScenarioLogger scenarioLogger;
private ScriptParams scenarioScriptParams;
private String scriptfile;
private Engine engine = Engine.Graalvm;
@ -110,6 +108,15 @@ public class Scenario implements Callable<ScenarioResult> {
this.wantsCompiledScript = wantsCompiledScript;
}
public Scenario setLogger(Logger logger) {
this.logger = logger;
return this;
}
public Logger getLogger() {
return logger;
}
public Scenario(String name, Engine engine) {
this.scenarioName = name;
this.engine = engine;
@ -353,10 +360,6 @@ public class Scenario implements Callable<ScenarioResult> {
return "name:'" + this.getScenarioName() + "'";
}
public void setScenarioLogger(ScenarioLogger scenarioLogger) {
this.scenarioLogger = scenarioLogger;
}
public void addScenarioScriptParams(ScriptParams scenarioScriptParams) {
this.scenarioScriptParams = scenarioScriptParams;
}

View File

@ -21,20 +21,17 @@ import io.nosqlbench.engine.core.IndexedThreadFactory;
import io.nosqlbench.engine.core.ScenarioController;
import io.nosqlbench.engine.core.ScenarioResult;
import io.nosqlbench.engine.core.ScenariosResults;
import io.nosqlbench.engine.core.logging.SessionLogConfig;
import io.nosqlbench.engine.core.logging.ScenarioLogger;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
public class ScenariosExecutor {
private final static Logger logger = LogManager.getLogger(ScenariosExecutor.class);
private final Logger logger = LogManager.getLogger("SCENARIOS");
private final LinkedHashMap<String, SubmittedScenario> submitted = new LinkedHashMap<>();
private final ExecutorService executor;
@ -54,15 +51,6 @@ public class ScenariosExecutor {
}
public synchronized void execute(Scenario scenario) {
ScenarioLogger logs = new SessionLogConfig(scenario.getScenarioName())
.setLogDir(Path.of("logs"))
.setMaxLogs(0)
.start();
execute(scenario, logs);
}
public synchronized void execute(Scenario scenario, ScenarioLogger scenarioLogger) {
scenario.setScenarioLogger(scenarioLogger);
if (submitted.get(scenario.getScenarioName()) != null) {
throw new BasicError("Scenario " + scenario.getScenarioName() + " is already defined. Remove it first to reuse the name.");
}

View File

@ -6,8 +6,6 @@ import io.nosqlbench.engine.cli.Cmd;
import io.nosqlbench.engine.cli.NBCLICommandParser;
import io.nosqlbench.engine.cli.ScriptBuffer;
import io.nosqlbench.engine.core.ScenarioResult;
import io.nosqlbench.engine.core.logging.SessionLogConfig;
import io.nosqlbench.engine.core.logging.ScenarioLogger;
import io.nosqlbench.engine.core.script.Scenario;
import io.nosqlbench.engine.core.script.ScenariosExecutor;
import io.nosqlbench.engine.rest.services.WorkSpace;
@ -115,11 +113,7 @@ public class ScenarioExecutorEndpoint implements WebServiceObject {
scenario.addScriptText(buffer.getParsedScript());
ScenarioLogger logger = new SessionLogConfig(scenario.getScenarioName())
.setLogDir(workspace.getWorkspacePath().resolve("logs"))
.start();
executor.execute(scenario, logger);
executor.execute(scenario);
return Response.created(UriBuilder.fromResource(ScenarioExecutorEndpoint.class).path(
"scenario/" + rq.getScenarioName()).build()).entity("started").build();

View File

@ -36,7 +36,7 @@ import java.util.regex.Pattern;
* invalid in this API, except when provided as a default value.
*/
public class Environment {
private final static Logger logger = LogManager.getLogger("ENVIRONMENT");
private Logger logger;
// package private for testing
Environment() {
@ -66,7 +66,9 @@ public class Environment {
}
if (references.containsKey(propname)) {
if (references.get(propname).equals(value)) {
logger.warn("changing already referenced property '" + propname + "' to same value");
if (logger != null) {
logger.warn("changing already referenced property '" + propname + "' to same value");
}
} else {
throw new BasicError("Changing already referenced property '" + propname + "' from \n" +
"'" + references.get(propname) + "' to '" + value + "' is not supported.\n" +
@ -108,7 +110,9 @@ public class Environment {
}
if (envToProp.containsKey(name.toUpperCase())) {
String propName = envToProp.get(name.toUpperCase());
logger.debug("redirecting env var '" + name + "' to property '" + propName + "'");
if (logger != null) {
logger.debug("redirecting env var '" + name + "' to property '" + propName + "'");
}
value = System.getProperty(propName);
if (value != null) {
return value;
@ -173,7 +177,9 @@ public class Environment {
}
String value = peek(envvar);
if (value == null) {
logger.debug("no value found for '" + envvar + "', returning Optional.empty() for '" + word + "'");
if (logger != null) {
logger.debug("no value found for '" + envvar + "', returning Optional.empty() for '" + word + "'");
}
return Optional.empty();
} else {
value = reference(envvar, value);

View File

@ -19,8 +19,6 @@ package io.nosqlbench.engine.core.script;
import io.nosqlbench.engine.core.ScenarioResult;
import io.nosqlbench.engine.core.ScenariosResults;
import io.nosqlbench.engine.core.logging.SessionLogConfig;
import io.nosqlbench.engine.core.logging.ScenarioLogger;
import org.apache.commons.compress.utils.IOUtils;
import org.assertj.core.data.Offset;
import org.junit.BeforeClass;
@ -30,7 +28,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
@ -69,11 +66,7 @@ public class AsyncScriptIntegrationTests {
}
s.addScriptText(script);
// s.addScriptText("load('classpath:scripts/async/" + scriptname + ".js');");
ScenarioLogger scenarioLogger = new SessionLogConfig(scenarioName)
.setMaxLogs(0)
.setLogDir(Path.of("logs/test"))
.start();
e.execute(s, scenarioLogger);
e.execute(s);
ScenariosResults scenariosResults = e.awaitAllResults();
ScenarioResult scenarioResult = scenariosResults.getOne();
scenarioResult.reportToLog();

View File

@ -19,8 +19,6 @@ package io.nosqlbench.engine.core.script;
import io.nosqlbench.engine.core.ScenarioResult;
import io.nosqlbench.engine.core.ScenariosResults;
import io.nosqlbench.engine.core.logging.SessionLogConfig;
import io.nosqlbench.engine.core.logging.ScenarioLogger;
import org.apache.commons.compress.utils.IOUtils;
import org.assertj.core.data.Offset;
import org.junit.BeforeClass;
@ -29,7 +27,6 @@ import org.junit.Test;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
@ -64,11 +61,7 @@ public class ScriptIntegrationTests {
}
s.addScriptText(script);
// s.addScriptText("load('classpath:scripts/sync/" + scriptname + ".js');");
ScenarioLogger scenarioLogger = new SessionLogConfig(scenarioName)
.setMaxLogs(0)
.setLogDir(Path.of("logs/test"))
.start();
e.execute(s, scenarioLogger);
e.execute(s);
ScenariosResults scenariosResults = e.awaitAllResults();
ScenarioResult scenarioResult = scenariosResults.getOne();
scenarioResult.reportToLog();