IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-12-23 15:40:44 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1601 from nosqlbench/client_metrics

Browse Source 
#1537: Sanity checking on client side load should be included into core metrics and async warnings
This commit is contained in:
Shaunak Das 2023-10-12 13:49:04 -07:00 committed by GitHub
commit 7ae60286ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 716 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -16,6 +16,8 @@
package io.nosqlbench.engine.cli;
import com.codahale.metrics.Gauge;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.api.annotations.Annotation;
@ -25,6 +27,7 @@ import io.nosqlbench.api.labels.NBLabels;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import io.nosqlbench.api.engine.metrics.instruments.NBFunctionGauge;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.logging.NBLogLevel;
import io.nosqlbench.api.metadata.SessionNamer;
@ -38,6 +41,12 @@ import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.cli.NBCLIOptions.LoggerConfigData;
import io.nosqlbench.engine.cli.NBCLIOptions.Mode;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.clientload.ClientSystemMetricChecker;
import io.nosqlbench.engine.core.clientload.DiskStatsReader;
import io.nosqlbench.engine.core.clientload.LoadAvgReader;
import io.nosqlbench.engine.core.clientload.MemInfoReader;
import io.nosqlbench.engine.core.clientload.NetDevReader;
import io.nosqlbench.engine.core.clientload.StatReader;
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
import io.nosqlbench.engine.core.lifecycle.process.ShutdownManager;
@ -88,6 +97,8 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
private String sessionCode;
private long sessionTime;
private ClientSystemMetricChecker clientMetricChecker;
public NBCLI(final String commandName) {
this.commandName = commandName;
}
@ -405,6 +416,15 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
final LoggerConfigData classicConfigs : options.getClassicHistoConfigs())
ActivityMetrics.addClassicHistos(sessionName, classicConfigs.pattern, classicConfigs.file, classicConfigs.interval);
// client machine metrics; TODO: modify pollInterval
this.clientMetricChecker = new ClientSystemMetricChecker(10);
registerLoadAvgMetrics();
registerMemInfoMetrics();
registerDiskStatsMetrics();
registerNetworkInterfaceMetrics();
registerStatMetrics();
clientMetricChecker.start();
// intentionally not shown for warn-only
NBCLI.logger.info(() -> "console logging level is " + options.getConsoleLogLevel());
@ -459,6 +479,7 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
final ScenariosResults scenariosResults = scenariosExecutor.awaitAllResults();
NBCLI.logger.debug(() -> "Total of " + scenariosResults.getSize() + " result object returned from ScenariosExecutor");
clientMetricChecker.shutdown();
ActivityMetrics.closeMetrics(options.wantsEnableChart());
scenariosResults.reportToLog();
ShutdownManager.shutdown();
@ -497,6 +518,125 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
return metrics;
}
private void registerLoadAvgMetrics() {
LoadAvgReader reader = new LoadAvgReader();
if (!reader.fileExists())
return;
Gauge<Double> loadAvgOneMinGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getOneMinLoadAverage(), "loadavg_1min")
);
Gauge<Double> loadAvgFiveMinGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getFiveMinLoadAverage(), "loadavg_5min")
);
Gauge<Double> loadAvgFifteenMinuteGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getFifteenMinLoadAverage(), "loadavg_15min")
);
// add checking for CPU load averages; TODO: Modify thresholds
clientMetricChecker.addMetricToCheck("loadavg_5min", loadAvgFiveMinGauge, 50.0);
clientMetricChecker.addMetricToCheck("loadavg_1min", loadAvgOneMinGauge, 50.0);
clientMetricChecker.addMetricToCheck("loadavg_15min", loadAvgFifteenMinuteGauge, 50.0);
}
private void registerMemInfoMetrics() {
MemInfoReader reader = new MemInfoReader();
if (!reader.fileExists())
return;
Gauge<Double> memTotalGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getMemTotalkB(), "mem_total")
);
Gauge<Double> memUsedGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getMemUsedkB(), "mem_used")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getMemFreekB(), "mem_free")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getMemAvailablekB(), "mem_available")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getMemCachedkB(), "mem_cached")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getMemBufferskB(), "mem_buffered")
);
// add checking for percent memory used at some given time; TODO: Modify percent threshold
clientMetricChecker.addRatioMetricToCheck(
"mem_used_percent", memUsedGauge, memTotalGauge, 90.0, false
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getSwapTotalkB(), "swap_total")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getSwapFreekB(), "swap_free")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getSwapUsedkB(), "swap_used")
);
}
private void registerDiskStatsMetrics() {
DiskStatsReader reader = new DiskStatsReader();
if (!reader.fileExists())
return;
for (String device: reader.getDevices()) {
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getTransactionsForDevice(device), device + "_transactions")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getKbReadForDevice(device), device + "_kB_read")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getKbWrittenForDevice(device), device + "_kB_written")
);
}
}
private void registerNetworkInterfaceMetrics() {
NetDevReader reader = new NetDevReader();
if (!reader.fileExists())
return;
for (String interfaceName: reader.getInterfaces()) {
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getBytesReceived(interfaceName), interfaceName + "_rx_bytes")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getPacketsReceived(interfaceName), interfaceName + "_rx_packets")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getBytesTransmitted(interfaceName), interfaceName + "_tx_bytes")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getPacketsTransmitted(interfaceName), interfaceName + "_tx_packets")
);
}
}
private void registerStatMetrics() {
StatReader reader = new StatReader();
if (!reader.fileExists())
return;
Gauge<Double> cpuUserGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getUserTime(), "cpu_user")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getSystemTime(), "cpu_system")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getIdleTime(), "cpu_idle")
);
ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getIoWaitTime(), "cpu_iowait")
);
Gauge<Double> cpuTotalGauge = ActivityMetrics.register(
new NBFunctionGauge(this, () -> reader.getTotalTime(), "cpu_total")
);
// add checking for percent of time spent in user space; TODO: Modify percent threshold
clientMetricChecker.addRatioMetricToCheck(
"cpu_user_percent", cpuUserGauge, cpuTotalGauge, 50.0, true
);
}
@Override
public NBLabels getLabels() {
return labels;

129
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/ClientSystemMetricChecker.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import com.codahale.metrics.Gauge;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.ArrayList;
import java.util.List;
public class ClientSystemMetricChecker {
private final int pollIntervalSeconds;
private final ScheduledExecutorService scheduler;
private List<ClientMetric> clientMetrics;
public ClientSystemMetricChecker(int pollIntervalSeconds) {
this.pollIntervalSeconds = pollIntervalSeconds;
this.scheduler = Executors.newScheduledThreadPool(1);
this.clientMetrics = new ArrayList<>();
}
public void addMetricToCheck(String name, Gauge<Double> metric, Double threshold) {
addRatioMetricToCheck(name, metric, null, threshold, false);
}
public void addRatioMetricToCheck(String name, Gauge<Double> numerator, Gauge<Double> denominator, Double threshold, boolean retainPrev) {
/**
* Some "meaningful" system metrics are derived via:
* - taking a ratio of instantaneous values (e.g. MemUsed / MemTotal from /proc/meminfo)
* - taking a ratio of deltas of aggregates values over a time window (e.g. CPU utilization from /proc/stat)
*
* This method serves to be able to allow checking those which can be derived as a ratio of two existing metrics.
*/
clientMetrics.add(new ClientMetric(name, numerator, denominator, threshold, retainPrev));
}
public void start() {
scheduler.scheduleAtFixedRate(() -> {
checkMetrics();
}, pollIntervalSeconds, pollIntervalSeconds, TimeUnit.SECONDS);
}
private void checkMetrics() {
for (ClientMetric c: clientMetrics)
c.check();
}
public void shutdown() {
scheduler.shutdown();
}
private class ClientMetric {
private static final Logger logger = LogManager.getLogger(ClientMetric.class);
private final String name;
private final Gauge<Double> numerator;
private final Gauge<Double> denominator;
private final Double threshold;
private final Boolean retainPrevValue;
private Double prevNumeratorValue;
private Double prevDenominatorValue;
private ClientMetric(String name, Gauge<Double> gauge, Double threshold) {
this(name, gauge, null, threshold, false);
}
private ClientMetric(String name, Gauge<Double> numerator, Gauge<Double> denominator, Double threshold, Boolean retainPrevValue) {
this.name = name;
this.numerator = numerator;
this.denominator = denominator;
this.threshold = threshold;
this.retainPrevValue = retainPrevValue;
this.prevNumeratorValue = null;
this.prevDenominatorValue = null;
}
private Double extract(){
Double numeratorVal = numerator.getValue();
if (numeratorVal == null)
return null;
Double deltaNumeratorVal = numeratorVal;
if (prevNumeratorValue != null)
deltaNumeratorVal -= prevNumeratorValue;
// the case that we are not extracting a ratio of values
if (denominator == null) {
if (retainPrevValue)
prevNumeratorValue = numeratorVal;
return deltaNumeratorVal;
}
// at this point, we will be extracting a ratio of gauge value changes over a time interval
Double denominatorVal = denominator.getValue();
if (denominatorVal == null)
return null;
Double deltaDenominatorVal = denominatorVal;
if (prevDenominatorValue != null)
deltaDenominatorVal -= prevDenominatorValue;
if (deltaDenominatorVal == 0.0)
return null;
Double percent = (deltaNumeratorVal / deltaDenominatorVal) * 100.0;
if (retainPrevValue) {
prevNumeratorValue = numeratorVal;
prevDenominatorValue = denominatorVal;
}
return percent;
}
private void check() {
Double extractedVal = extract();
if (extractedVal != null && extractedVal > threshold)
logger.warn(name + " value = " + extractedVal + " > threshold " + threshold);
}
}
}

82
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/DiskStatsReader.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class DiskStatsReader extends LinuxSystemFileReader {
/**
* Note that all fields are cumulative within /proc/diskstats.
*
* Reference:
* - https://serverfault.com/questions/619097/interpreting-proc-diskstats-for-a-webserver-more-writes-than-reads
*
* Example line:
* 259 0 nvme0n1 669494 21 65326120 388760 3204963 2891102 734524354 42209620 0 446420 41361212
*/
private static final Double sectorSizeBytes = 512.0;
public DiskStatsReader() {
super("/proc/diskstats");
}
public Double getTransactionsForDevice(String deviceName) {
MatchResult result = findFirstMatch(Pattern.compile(buildRegex(deviceName)));
if (result == null)
return null;
Double readsCompleted = Double.valueOf(result.group(1));
Double writesCompleted = Double.valueOf(result.group(5));
return readsCompleted + writesCompleted;
}
public Double getKbReadForDevice(String deviceName) {
MatchResult result = findFirstMatch(Pattern.compile(buildRegex(deviceName)));
if (result == null)
return null;
Double sectorsRead = Double.valueOf(result.group(3));
return sectorsRead * sectorSizeBytes / 1024;
}
public Double getKbWrittenForDevice(String deviceName) {
MatchResult result = findFirstMatch(Pattern.compile(buildRegex(deviceName)));
if (result == null)
return null;
Double sectorsWritten = Double.valueOf(result.group(7));
return sectorsWritten * sectorSizeBytes / 1024;
}
private String buildRegex(String deviceName) {
return "\\b" + Pattern.quote(deviceName) + "\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)";
}
public List<String> getDevices() {
String regex = "^\\s*\\d+\\s+\\d+\\s+([a-zA-Z0-9]+)\\s+.*$";
Pattern pattern = Pattern.compile(regex);
List<MatchResult> results = findAllLinesMatching(pattern);
return results.stream().map(m -> m.group(1)).collect(Collectors.toList());
}
}

99
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LinuxSystemFileReader.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.Files;
import java.util.List;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
public abstract class LinuxSystemFileReader {
protected Logger logger;
protected String filePath;
public LinuxSystemFileReader(String filePath) {
logger = LogManager.getLogger(this.getClass());
this.filePath = filePath;
}
public boolean fileExists() {
Path path = Paths.get(filePath);
return Files.exists(path);
}
protected Double extract(String regex, int groupIdx){
Pattern pattern = Pattern.compile(regex);
MatchResult result = findFirstMatch(pattern);
if (result == null)
return null;
assert (1 <= groupIdx && groupIdx <= result.groupCount());
return Double.valueOf(result.group(groupIdx));
}
protected MatchResult findFirstMatch(Pattern pattern) {
Matcher matcher = null;
try (FileReader file = new FileReader(filePath);
BufferedReader reader = new BufferedReader(file)) {
String line;
while ((line = reader.readLine()) != null) {
matcher = pattern.matcher(line);
if (matcher.find())
break;
}
} catch (FileNotFoundException e) {
logger.warn("File not found: " + filePath);
} catch (final Throwable t) {
throw new RuntimeException("Failed to read " + filePath);
}
if (matcher == null)
return null;
return matcher.toMatchResult();
}
protected List<MatchResult> findAllLinesMatching(Pattern pattern) {
List<MatchResult> results = new ArrayList<>();
Matcher matcher;
try (FileReader file = new FileReader(filePath);
BufferedReader reader = new BufferedReader(file)) {
String line;
while ((line = reader.readLine()) != null) {
try {
matcher = pattern.matcher(line);
if (matcher.find())
results.add(matcher.toMatchResult());
} catch (Exception e) {
logger.error("Error processing line: " + e.getMessage());
}
}
} catch (FileNotFoundException e) {
logger.warn("File not found: " + filePath);
}
catch (final Throwable t) {
throw new RuntimeException("Failed to read " + filePath);
}
return results;
}
}

44
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LoadAvgReader.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class LoadAvgReader extends LinuxSystemFileReader {
/**
* Example line:
* 0.78 1.39 2.03 1/2153 2818
*/
private static final String regex = "(\\d+\\.\\d+)\\s(\\d+\\.\\d+)\\s(\\d+\\.\\d+)";
public LoadAvgReader(){
super("/proc/loadavg");
}
public Double getOneMinLoadAverage() {
return extract(regex, 1);
}
public Double getFiveMinLoadAverage() {
return extract(regex, 2);
}
public Double getFifteenMinLoadAverage() {
return extract(regex, 3);
}
}

82
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/MemInfoReader.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class MemInfoReader extends LinuxSystemFileReader {
/**
* References:
* - https://docs.kernel.org/filesystems/proc.html#meminfo
* - https://stackoverflow.com/questions/41224738/how-to-calculate-system-memory-usage-from-proc-meminfo-like-htop
*/
public MemInfoReader() {
super("/proc/meminfo");
}
public Double getMemTotalkB() {
String regex = "MemTotal:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getMemFreekB() {
String regex = "MemFree:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getMemAvailablekB() {
String regex = "MemAvailable:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getMemUsedkB() {
Double memTotal = getMemTotalkB();
Double memFree = getMemFreekB();
if (memTotal != null && memFree != null)
return memTotal - memFree;
return null;
}
public Double getMemCachedkB() {
String regex = "Cached:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getMemBufferskB() {
String regex = "Buffers:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getSwapTotalkB() {
String regex = "SwapTotal:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getSwapFreekB() {
String regex = "SwapFree:\\s+(\\d+) kB";
return extract(regex, 1);
}
public Double getSwapUsedkB() {
Double swapTotal = getSwapTotalkB();
Double swapFree = getSwapFreekB();
if (swapTotal != null && swapFree != null)
return swapTotal - swapFree;
return null;
}
}

71
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/NetDevReader.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class NetDevReader extends LinuxSystemFileReader {
/**
* Note that all fields are cumulative in /proc/net/dev
*
* Reference:
* - https://www.linuxquestions.org/questions/linux-networking-3/need-explanation-of-proc-net-dev-bytes-counters-4175458860/
*
* Example line:
* wlp59s0: 2941956695 4935327 0 0 0 0 0 0 1213470966 3450551 0 0 0 0 0 0
*/
public NetDevReader() {
super("/proc/net/dev");
}
public Double getBytesReceived(String interfaceName) {
return extract(buildRegex(interfaceName), 1);
}
public Double getPacketsReceived(String interfaceName) {
return extract(buildRegex(interfaceName), 2);
}
public Double getBytesTransmitted(String interfaceName) {
return extract(buildRegex(interfaceName), 3);
}
public Double getPacketsTransmitted(String interfaceName) {
return extract(buildRegex(interfaceName), 4);
}
private String buildRegex(String interfaceName) {
return "\\b" + Pattern.quote(interfaceName) + "\\s*:(\\s*\\d+)\\s+(\\d+)\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+(\\d+)\\s+(\\d+)";
}
public List<String> getInterfaces() {
String regex = "^\\s*([^\\s:]+):\\s*\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+";
Pattern pattern = Pattern.compile(regex);
List<MatchResult> results = findAllLinesMatching(pattern);
return results.stream().map(m -> m.group(1)).collect(Collectors.toList());
}
}

69
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/StatReader.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.core.clientload;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
public class StatReader extends LinuxSystemFileReader {
/**
* Note that all fields are cumulative within /proc/stat.
*
* Reference:
* - https://docs.kernel.org/filesystems/proc.html#miscellaneous-kernel-statistics-in-proc-stat
*
* Example line:
* cpu 6955150 945 1205506 139439365 115574 0 113356 0 0 0
*/
private static final String regex = "cpu\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)";
public StatReader() {
super("/proc/stat");
}
public Double getUserTime() {
return extract(regex, 1);
}
public Double getSystemTime() {
return extract(regex, 3);
}
public Double getIdleTime() {
return extract(regex, 4);
}
public Double getIoWaitTime() {
return extract(regex, 5);
}
public Double getTotalTime() {
MatchResult result = findFirstMatch(Pattern.compile(regex));
if (result == null)
return null;
Double user = Double.valueOf(result.group(1));
Double nice = Double.valueOf(result.group(2));
Double system = Double.valueOf(result.group(3));
Double idle = Double.valueOf(result.group(4));
Double iowait = Double.valueOf(result.group(5));
Double irq = Double.valueOf(result.group(6));
Double softirq = Double.valueOf(result.group(7));
return user + nice + system + idle + iowait + irq + softirq;
}
}