mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
enable separate configuration of logging patterns
This commit is contained in:
Jonathan Shook
parent
c3544094b3
commit
d223e4c6c5
@ -97,14 +97,16 @@ public class NBCLI {
|
||||
String sessionName = SessionNamer.format(globalOptions.getSessionName());
|
||||
|
||||
loggerConfig
|
||||
.setSessionName(sessionName)
|
||||
.setConsoleLevel(globalOptions.getConsoleLogLevel())
|
||||
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
|
||||
.setLogfileLevel(globalOptions.getScenarioLogLevel())
|
||||
.getLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
|
||||
.setMaxLogs(globalOptions.getLogsMax())
|
||||
.setLogsDirectory(globalOptions.getLogsDirectory())
|
||||
.activate();
|
||||
.setSessionName(sessionName)
|
||||
.setConsoleLevel(globalOptions.getConsoleLogLevel())
|
||||
.setConsolePattern(globalOptions.getConsoleLoggingPattern())
|
||||
.setLogfileLevel(globalOptions.getScenarioLogLevel())
|
||||
.setLogfilePattern(globalOptions.getLogfileLoggingPattern())
|
||||
.getLoggerLevelOverrides(globalOptions.getLogLevelOverrides())
|
||||
.setMaxLogs(globalOptions.getLogsMax())
|
||||
.setLogsDirectory(globalOptions.getLogsDirectory())
|
||||
.setAnsiEnabled(globalOptions.isEnableAnsi())
|
||||
.activate();
|
||||
ConfigurationFactory.setConfigurationFactory(loggerConfig);
|
||||
|
||||
logger = LogManager.getLogger("NBCLI");
|
||||
@ -119,7 +121,7 @@ public class NBCLI {
|
||||
}
|
||||
|
||||
logger.info("Running NoSQLBench Version " + new VersionInfo().getVersion());
|
||||
logger.info("command-line: "+Arrays.stream(args).collect(Collectors.joining(" ")));
|
||||
logger.info("command-line: " + Arrays.stream(args).collect(Collectors.joining(" ")));
|
||||
logger.info("client-hardware: " + SystemId.getHostSummary());
|
||||
|
||||
boolean dockerMetrics = globalOptions.wantsDockerMetrics();
|
||||
@ -130,10 +132,10 @@ public class NBCLI {
|
||||
int mOpts = (dockerMetrics ? 1 : 0) + (dockerMetricsAt != null ? 1 : 0) + (reportGraphiteTo != null ? 1 : 0);
|
||||
if (mOpts > 1 && (reportGraphiteTo == null || annotatorsConfig == null)) {
|
||||
throw new BasicError("You have multiple conflicting options which attempt to set\n" +
|
||||
" the destination for metrics and annotations. Please select only one of\n" +
|
||||
" --docker-metrics, --docker-metrics-at <addr>, or other options like \n" +
|
||||
" --report-graphite-to <addr> and --annotators <config>\n" +
|
||||
" For more details, see run 'nb help docker-metrics'");
|
||||
" the destination for metrics and annotations. Please select only one of\n" +
|
||||
" --docker-metrics, --docker-metrics-at <addr>, or other options like \n" +
|
||||
" --report-graphite-to <addr> and --annotators <config>\n" +
|
||||
" For more details, see run 'nb help docker-metrics'");
|
||||
}
|
||||
|
||||
String metricsAddr = null;
|
||||
@ -143,13 +145,13 @@ public class NBCLI {
|
||||
logger.info("Docker metrics is enabled. Docker must be installed for this to work");
|
||||
DockerMetricsManager dmh = new DockerMetricsManager();
|
||||
Map<String, String> dashboardOptions = Map.of(
|
||||
DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag(),
|
||||
DockerMetricsManager.PROM_TAG, globalOptions.getDockerPromTag(),
|
||||
DockerMetricsManager.TSDB_RETENTION, String.valueOf(globalOptions.getDockerPromRetentionDays())
|
||||
DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag(),
|
||||
DockerMetricsManager.PROM_TAG, globalOptions.getDockerPromTag(),
|
||||
DockerMetricsManager.TSDB_RETENTION, String.valueOf(globalOptions.getDockerPromRetentionDays())
|
||||
);
|
||||
dmh.startMetrics(dashboardOptions);
|
||||
String warn = "Docker Containers are started, for grafana and prometheus, hit" +
|
||||
" these urls in your browser: http://<host>:3000 and http://<host>:9090";
|
||||
" these urls in your browser: http://<host>:3000 and http://<host>:9090";
|
||||
logger.warn(warn);
|
||||
metricsAddr = "localhost";
|
||||
} else if (dockerMetricsAt != null) {
|
||||
@ -159,8 +161,8 @@ public class NBCLI {
|
||||
if (metricsAddr != null) {
|
||||
reportGraphiteTo = metricsAddr + ":9109";
|
||||
annotatorsConfig = "[{type:'log',level:'info'},{type:'grafana',baseurl:'http://" + metricsAddr + ":3000" +
|
||||
"/'," +
|
||||
"tags:'appname:nosqlbench',timeoutms:5000,onerror:'warn'}]";
|
||||
"/'," +
|
||||
"tags:'appname:nosqlbench',timeoutms:5000,onerror:'warn'}]";
|
||||
} else {
|
||||
annotatorsConfig = "[{type:'log',level:'info'}]";
|
||||
}
|
||||
@ -225,22 +227,22 @@ public class NBCLI {
|
||||
logger.debug("user requests to copy out " + resourceToCopy);
|
||||
|
||||
Optional<Content<?>> tocopy = NBIO.classpath()
|
||||
.prefix("activities")
|
||||
.prefix(options.wantsIncludes())
|
||||
.name(resourceToCopy).extension(RawStmtsLoader.YAML_EXTENSIONS).first();
|
||||
.prefix("activities")
|
||||
.prefix(options.wantsIncludes())
|
||||
.name(resourceToCopy).extension(RawStmtsLoader.YAML_EXTENSIONS).first();
|
||||
|
||||
if (tocopy.isEmpty()) {
|
||||
|
||||
tocopy = NBIO.classpath()
|
||||
.prefix().prefix(options.wantsIncludes())
|
||||
.prefix(options.wantsIncludes())
|
||||
.name(resourceToCopy).first();
|
||||
.prefix().prefix(options.wantsIncludes())
|
||||
.prefix(options.wantsIncludes())
|
||||
.name(resourceToCopy).first();
|
||||
}
|
||||
|
||||
Content<?> data = tocopy.orElseThrow(
|
||||
() -> new BasicError(
|
||||
"Unable to find " + resourceToCopy +
|
||||
" in classpath to copy out")
|
||||
() -> new BasicError(
|
||||
"Unable to find " + resourceToCopy +
|
||||
" in classpath to copy out")
|
||||
);
|
||||
|
||||
Path writeTo = Path.of(data.asPath().getFileName().toString());
|
||||
@ -280,7 +282,7 @@ public class NBCLI {
|
||||
if (options.wantsTopicalHelp()) {
|
||||
Optional<String> helpDoc = MarkdownDocInfo.forHelpTopic(options.wantsTopicalHelpFor());
|
||||
System.out.println(helpDoc.orElseThrow(
|
||||
() -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
|
||||
() -> new RuntimeException("No help could be found for " + options.wantsTopicalHelpFor())
|
||||
));
|
||||
System.exit(0);
|
||||
}
|
||||
@ -328,15 +330,15 @@ public class NBCLI {
|
||||
}
|
||||
|
||||
for (
|
||||
NBCLIOptions.LoggerConfigData histoLogger : options.getHistoLoggerConfigs()) {
|
||||
NBCLIOptions.LoggerConfigData histoLogger : options.getHistoLoggerConfigs()) {
|
||||
ActivityMetrics.addHistoLogger(sessionName, histoLogger.pattern, histoLogger.file, histoLogger.interval);
|
||||
}
|
||||
for (
|
||||
NBCLIOptions.LoggerConfigData statsLogger : options.getStatsLoggerConfigs()) {
|
||||
NBCLIOptions.LoggerConfigData statsLogger : options.getStatsLoggerConfigs()) {
|
||||
ActivityMetrics.addStatsLogger(sessionName, statsLogger.pattern, statsLogger.file, statsLogger.interval);
|
||||
}
|
||||
for (
|
||||
NBCLIOptions.LoggerConfigData classicConfigs : options.getClassicHistoConfigs()) {
|
||||
NBCLIOptions.LoggerConfigData classicConfigs : options.getClassicHistoConfigs()) {
|
||||
ActivityMetrics.addClassicHistos(sessionName, classicConfigs.pattern, classicConfigs.file, classicConfigs.interval);
|
||||
}
|
||||
|
||||
|
||||
@ -34,7 +34,6 @@ public class NBCLIOptions {
|
||||
|
||||
|
||||
private static final String METRICS_PREFIX = "--metrics-prefix";
|
||||
|
||||
private static final String ANNOTATE_EVENTS = "--annotate";
|
||||
private static final String ANNOTATORS_CONFIG = "--annotators";
|
||||
|
||||
@ -83,6 +82,9 @@ public class NBCLIOptions {
|
||||
private final static String REPORT_SUMMARY_TO_DEFAULT = "stdout:60,_LOGS_/_SESSION_.summary";
|
||||
private static final String PROGRESS = "--progress";
|
||||
private static final String WITH_LOGGING_PATTERN = "--with-logging-pattern";
|
||||
private static final String LOGGING_PATTERN = "--logging-pattern";
|
||||
private static final String CONSOLE_PATTERN = "--console-pattern";
|
||||
private static final String LOGFILE_PATTERN = "--logfile-pattern";
|
||||
private static final String LOG_HISTOGRAMS = "--log-histograms";
|
||||
private static final String LOG_HISTOSTATS = "--log-histostats";
|
||||
private static final String CLASSIC_HISTOGRAMS = "--classic-histograms";
|
||||
@ -99,7 +101,8 @@ public class NBCLIOptions {
|
||||
private static final String NASHORN_ENGINE = "--nashorn";
|
||||
private static final String GRAALJS_COMPAT = "--graaljs-compat";
|
||||
|
||||
private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%7r %-5level [%t] %-12logger{0} %msg%n%throwable";
|
||||
private static final String DEFAULT_CONSOLE_PATTERN = "TERSE";
|
||||
private static final String DEFAULT_LOGFILE_PATTERN = "VERBOSE";
|
||||
|
||||
// private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
|
||||
public static final String NBSTATEDIR = "NBSTATEDIR";
|
||||
@ -130,7 +133,8 @@ public class NBCLIOptions {
|
||||
private boolean wantsMarkerTypes = false;
|
||||
private String[] rleDumpOptions = new String[0];
|
||||
private String[] cyclelogImportOptions = new String[0];
|
||||
private String consoleLoggingPattern = DEFAULT_CONSOLE_LOGGING_PATTERN;
|
||||
private String consoleLoggingPattern = DEFAULT_CONSOLE_PATTERN;
|
||||
private String logfileLoggingPattern = DEFAULT_LOGFILE_PATTERN;
|
||||
private NBLogLevel logsLevel = NBLogLevel.INFO;
|
||||
private Map<String, String> logLevelsOverrides = new HashMap<>();
|
||||
private boolean enableChart = false;
|
||||
@ -369,9 +373,20 @@ public class NBCLIOptions {
|
||||
arglist.removeFirst();
|
||||
logLevelsOverrides = parseLogLevelOverrides(readWordOrThrow(arglist, "log levels in name:LEVEL,... format"));
|
||||
break;
|
||||
case WITH_LOGGING_PATTERN:
|
||||
case CONSOLE_PATTERN:
|
||||
arglist.removeFirst();
|
||||
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
|
||||
consoleLoggingPattern =readWordOrThrow(arglist, "console pattern");
|
||||
break;
|
||||
case LOGFILE_PATTERN:
|
||||
arglist.removeFirst();
|
||||
logfileLoggingPattern =readWordOrThrow(arglist, "logfile pattern");
|
||||
break;
|
||||
case WITH_LOGGING_PATTERN:
|
||||
case LOGGING_PATTERN:
|
||||
arglist.removeFirst();
|
||||
String pattern = readWordOrThrow(arglist, "console and logfile pattern");
|
||||
consoleLoggingPattern = pattern;
|
||||
logfileLoggingPattern = pattern;
|
||||
break;
|
||||
default:
|
||||
nonincludes.addLast(arglist.removeFirst());
|
||||
|
||||
@ -28,9 +28,31 @@ import java.util.stream.Collectors;
|
||||
|
||||
//@Plugin(name = "CustomConfigurationFactory", category = ConfigurationFactory.CATEGORY)
|
||||
//@Order(50)
|
||||
// Can't use plugin injection, since we need a tailored instance before logging
|
||||
// Can't use plugin injection, since we need a tailored instance before logging is fully initialized
|
||||
|
||||
/**
|
||||
* This is a custom programmatic logger config handler which allows for a variety of
|
||||
* logging features to be controlled at runtime.
|
||||
*
|
||||
* @see <a href="https://logging.apache.org/log4j/2.x/manual/layouts.html#Pattern_Layout">Pattern Layout</a>
|
||||
*/
|
||||
public class LoggerConfig extends ConfigurationFactory {
|
||||
|
||||
public static Map<String, String> STANDARD_FORMATS = Map.of(
|
||||
"TERSE", "%8r %-5level [%t] %-12logger{0} %msg%n%throwable",
|
||||
"VERBOSE", "%d{DEFAULT}{GMT} [%t] %logger %-5level: %msg%n%throwable",
|
||||
"TERSE-ANSI", "%8r %highlight{%-5level} %style{%C{1.} [%t] %-12logger{0}} %msg%n%throwable",
|
||||
"VERBOSE-ANSI", "%d{DEFAULT}{GMT} [%t] %highlight{%logger %-5level}: %msg%n%throwable"
|
||||
);
|
||||
|
||||
/**
|
||||
* Some included libraries are spammy and intefere with normal diagnostic visibility, so
|
||||
* we squelch them to some reasonable level so they aren't a nuisance.
|
||||
*/
|
||||
public static Map<String, Level> BUILTIN_OVERRIDES = Map.of(
|
||||
"oshi.util", Level.INFO
|
||||
);
|
||||
|
||||
/**
|
||||
* ArgsFile
|
||||
* Environment
|
||||
@ -50,11 +72,17 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
private String sessionName;
|
||||
private int maxLogfiles = 100;
|
||||
private String logfileLocation;
|
||||
private boolean ansiEnabled;
|
||||
|
||||
|
||||
public LoggerConfig() {
|
||||
}
|
||||
|
||||
public LoggerConfig setAnsiEnabled(boolean ansiEnabled) {
|
||||
this.ansiEnabled = ansiEnabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
public LoggerConfig setConsoleLevel(NBLogLevel level) {
|
||||
this.consoleLevel = level;
|
||||
return this;
|
||||
@ -97,20 +125,20 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
builder.setStatusLevel(internalLoggingStatusThreshold);
|
||||
|
||||
builder.add(
|
||||
builder.newFilter(
|
||||
"ThresholdFilter",
|
||||
Filter.Result.ACCEPT,
|
||||
Filter.Result.NEUTRAL
|
||||
).addAttribute("level", builderThresholdLevel)
|
||||
builder.newFilter(
|
||||
"ThresholdFilter",
|
||||
Filter.Result.ACCEPT,
|
||||
Filter.Result.NEUTRAL
|
||||
).addAttribute("level", builderThresholdLevel)
|
||||
);
|
||||
|
||||
// CONSOLE appender
|
||||
AppenderComponentBuilder appenderBuilder =
|
||||
builder.newAppender("console", "CONSOLE")
|
||||
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
|
||||
builder.newAppender("console", "CONSOLE")
|
||||
.addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT);
|
||||
|
||||
appenderBuilder.add(builder.newLayout("PatternLayout")
|
||||
.addAttribute("pattern", consolePattern));
|
||||
.addAttribute("pattern", consolePattern));
|
||||
|
||||
// appenderBuilder.add(
|
||||
// builder.newFilter("MarkerFilter", Filter.Result.DENY, Filter.Result.NEUTRAL)
|
||||
@ -120,8 +148,8 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
|
||||
// Log4J internal logging
|
||||
builder.add(builder.newLogger("org.apache.logging.log4j", Level.DEBUG)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.addAttribute("additivity", false));
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.addAttribute("additivity", false));
|
||||
|
||||
if (sessionName != null) {
|
||||
|
||||
@ -135,51 +163,56 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
|
||||
// LOGFILE appender
|
||||
LayoutComponentBuilder logfileLayout = builder.newLayout("PatternLayout")
|
||||
.addAttribute("pattern", logfilePattern);
|
||||
.addAttribute("pattern", logfilePattern);
|
||||
|
||||
String filebase = getSessionName().replaceAll("\\s", "_");
|
||||
String logfilePath = loggerDir.resolve(filebase + ".log").toString();
|
||||
this.logfileLocation = logfilePath;
|
||||
String archivePath = loggerDir.resolve(filebase + "-TIMESTAMP.log.gz").toString()
|
||||
.replaceAll("TIMESTAMP", "%d{MM-dd-yy}");
|
||||
.replaceAll("TIMESTAMP", "%d{MM-dd-yy}");
|
||||
|
||||
ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
|
||||
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
|
||||
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
|
||||
.addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
|
||||
.addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "100M"));
|
||||
|
||||
AppenderComponentBuilder logsAppenderBuilder =
|
||||
builder.newAppender("SCENARIO_APPENDER", RollingFileAppender.PLUGIN_NAME)
|
||||
.addAttribute("fileName", logfilePath)
|
||||
.addAttribute("filePattern", archivePath)
|
||||
.addAttribute("append", false)
|
||||
.add(logfileLayout)
|
||||
.addComponent(triggeringPolicy);
|
||||
builder.newAppender("SCENARIO_APPENDER", RollingFileAppender.PLUGIN_NAME)
|
||||
.addAttribute("fileName", logfilePath)
|
||||
.addAttribute("filePattern", archivePath)
|
||||
.addAttribute("append", false)
|
||||
.add(logfileLayout)
|
||||
.addComponent(triggeringPolicy);
|
||||
builder.add(logsAppenderBuilder);
|
||||
|
||||
rootBuilder.add(
|
||||
builder.newAppenderRef("SCENARIO_APPENDER")
|
||||
.addAttribute("level", Level.valueOf(getEffectiveFileLevel().toString()))
|
||||
builder.newAppenderRef("SCENARIO_APPENDER")
|
||||
.addAttribute("level", Level.valueOf(getEffectiveFileLevel().toString()))
|
||||
);
|
||||
}
|
||||
|
||||
rootBuilder.add(
|
||||
builder.newAppenderRef("console")
|
||||
.addAttribute("level",
|
||||
Level.valueOf(consoleLevel.toString())
|
||||
)
|
||||
builder.newAppenderRef("console")
|
||||
.addAttribute("level",
|
||||
Level.valueOf(consoleLevel.toString())
|
||||
)
|
||||
);
|
||||
|
||||
builder.add(rootBuilder);
|
||||
|
||||
if (logLevelOverrides != null) {
|
||||
logLevelOverrides.forEach((k, v) -> {
|
||||
Level olevel = Level.valueOf(v);
|
||||
builder.add(builder.newLogger(k, olevel)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.add(builder.newAppenderRef("SCENARIO_APPENDER"))
|
||||
.addAttribute("additivity", true));
|
||||
});
|
||||
}
|
||||
BUILTIN_OVERRIDES.forEach((k, v) -> {
|
||||
builder.add(builder.newLogger(k, v)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.add(builder.newAppenderRef("SCENARIO_APPENDER"))
|
||||
.addAttribute("additivity", true));
|
||||
});
|
||||
|
||||
logLevelOverrides.forEach((k, v) -> {
|
||||
Level olevel = Level.valueOf(v);
|
||||
builder.add(builder.newLogger(k, olevel)
|
||||
.add(builder.newAppenderRef("console"))
|
||||
.add(builder.newAppenderRef("SCENARIO_APPENDER"))
|
||||
.addAttribute("additivity", true));
|
||||
});
|
||||
|
||||
BuiltConfiguration builtConfig = builder.build();
|
||||
return builtConfig;
|
||||
@ -209,7 +242,7 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
if (!Files.exists(loggerDir)) {
|
||||
try {
|
||||
FileAttribute<Set<PosixFilePermission>> attrs = PosixFilePermissions.asFileAttribute(
|
||||
PosixFilePermissions.fromString("rwxrwx---")
|
||||
PosixFilePermissions.fromString("rwxrwx---")
|
||||
);
|
||||
Path directory = Files.createDirectory(loggerDir, attrs);
|
||||
} catch (Exception e) {
|
||||
@ -220,7 +253,19 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
}
|
||||
|
||||
public LoggerConfig setConsolePattern(String consoleLoggingPattern) {
|
||||
this.consolePattern = consoleLoggingPattern;
|
||||
|
||||
consoleLoggingPattern= (ansiEnabled && STANDARD_FORMATS.containsKey(consoleLoggingPattern+"-ANSI"))
|
||||
? consoleLoggingPattern+"-ANSI" : consoleLoggingPattern;
|
||||
|
||||
this.consolePattern = STANDARD_FORMATS.getOrDefault(consoleLoggingPattern, consoleLoggingPattern);
|
||||
return this;
|
||||
}
|
||||
|
||||
public LoggerConfig setLogfilePattern(String logfileLoggingPattern) {
|
||||
logfileLoggingPattern= (logfileLoggingPattern.endsWith("-ANSI") && STANDARD_FORMATS.containsKey(logfileLoggingPattern))
|
||||
? logfileLoggingPattern.substring(logfileLoggingPattern.length()-5) : logfileLoggingPattern;
|
||||
|
||||
this.logfileLocation = STANDARD_FORMATS.getOrDefault(logfileLoggingPattern, logfileLoggingPattern);
|
||||
return this;
|
||||
}
|
||||
|
||||
@ -263,9 +308,9 @@ public class LoggerConfig extends ConfigurationFactory {
|
||||
}
|
||||
|
||||
List<File> toDelete = filesList.stream()
|
||||
.sorted(fileTimeComparator)
|
||||
.limit(remove)
|
||||
.collect(Collectors.toList());
|
||||
.sorted(fileTimeComparator)
|
||||
.limit(remove)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (File file : toDelete) {
|
||||
logger.info("removing extra logfile: " + file.getPath());
|
||||
|
||||
Loading…
Reference in New Issue
Block a user