diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 9d637bc26..6905c91b0 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -16,6 +16,8 @@ package io.nosqlbench.engine.cli; + +import com.codahale.metrics.Gauge; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.nosqlbench.api.annotations.Annotation; @@ -25,6 +27,7 @@ import io.nosqlbench.api.labels.NBLabels; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; import io.nosqlbench.api.engine.metrics.ActivityMetrics; +import io.nosqlbench.api.engine.metrics.instruments.NBFunctionGauge; import io.nosqlbench.api.errors.BasicError; import io.nosqlbench.api.logging.NBLogLevel; import io.nosqlbench.api.metadata.SessionNamer; @@ -38,6 +41,12 @@ import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsLoader; import io.nosqlbench.engine.cli.NBCLIOptions.LoggerConfigData; import io.nosqlbench.engine.cli.NBCLIOptions.Mode; import io.nosqlbench.engine.core.annotation.Annotators; +import io.nosqlbench.engine.core.clientload.ClientSystemMetricChecker; +import io.nosqlbench.engine.core.clientload.DiskStatsReader; +import io.nosqlbench.engine.core.clientload.LoadAvgReader; +import io.nosqlbench.engine.core.clientload.MemInfoReader; +import io.nosqlbench.engine.core.clientload.NetDevReader; +import io.nosqlbench.engine.core.clientload.StatReader; import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler; import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader; import io.nosqlbench.engine.core.lifecycle.process.ShutdownManager; @@ -88,6 +97,8 @@ public class NBCLI implements Function, NBLabeledElement { private String sessionCode; private long sessionTime; + private ClientSystemMetricChecker clientMetricChecker; + public NBCLI(final String commandName) { this.commandName = commandName; } @@ -405,6 +416,15 @@ public class NBCLI implements Function, NBLabeledElement { final LoggerConfigData classicConfigs : options.getClassicHistoConfigs()) ActivityMetrics.addClassicHistos(sessionName, classicConfigs.pattern, classicConfigs.file, classicConfigs.interval); + // client machine metrics; TODO: modify pollInterval + this.clientMetricChecker = new ClientSystemMetricChecker(10); + registerLoadAvgMetrics(); + registerMemInfoMetrics(); + registerDiskStatsMetrics(); + registerNetworkInterfaceMetrics(); + registerStatMetrics(); + clientMetricChecker.start(); + // intentionally not shown for warn-only NBCLI.logger.info(() -> "console logging level is " + options.getConsoleLogLevel()); @@ -459,6 +479,7 @@ public class NBCLI implements Function, NBLabeledElement { final ScenariosResults scenariosResults = scenariosExecutor.awaitAllResults(); NBCLI.logger.debug(() -> "Total of " + scenariosResults.getSize() + " result object returned from ScenariosExecutor"); + clientMetricChecker.shutdown(); ActivityMetrics.closeMetrics(options.wantsEnableChart()); scenariosResults.reportToLog(); ShutdownManager.shutdown(); @@ -497,6 +518,125 @@ public class NBCLI implements Function, NBLabeledElement { return metrics; } + private void registerLoadAvgMetrics() { + LoadAvgReader reader = new LoadAvgReader(); + if (!reader.fileExists()) + return; + Gauge loadAvgOneMinGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getOneMinLoadAverage(), "loadavg_1min") + ); + Gauge loadAvgFiveMinGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getFiveMinLoadAverage(), "loadavg_5min") + ); + Gauge loadAvgFifteenMinuteGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getFifteenMinLoadAverage(), "loadavg_15min") + ); + // add checking for CPU load averages; TODO: Modify thresholds + clientMetricChecker.addMetricToCheck("loadavg_5min", loadAvgFiveMinGauge, 50.0); + clientMetricChecker.addMetricToCheck("loadavg_1min", loadAvgOneMinGauge, 50.0); + clientMetricChecker.addMetricToCheck("loadavg_15min", loadAvgFifteenMinuteGauge, 50.0); + } + + private void registerMemInfoMetrics() { + MemInfoReader reader = new MemInfoReader(); + if (!reader.fileExists()) + return; + Gauge memTotalGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getMemTotalkB(), "mem_total") + ); + Gauge memUsedGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getMemUsedkB(), "mem_used") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getMemFreekB(), "mem_free") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getMemAvailablekB(), "mem_available") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getMemCachedkB(), "mem_cached") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getMemBufferskB(), "mem_buffered") + ); + // add checking for percent memory used at some given time; TODO: Modify percent threshold + clientMetricChecker.addRatioMetricToCheck( + "mem_used_percent", memUsedGauge, memTotalGauge, 90.0, false + ); + + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getSwapTotalkB(), "swap_total") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getSwapFreekB(), "swap_free") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getSwapUsedkB(), "swap_used") + ); + } + + private void registerDiskStatsMetrics() { + DiskStatsReader reader = new DiskStatsReader(); + if (!reader.fileExists()) + return; + for (String device: reader.getDevices()) { + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getTransactionsForDevice(device), device + "_transactions") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getKbReadForDevice(device), device + "_kB_read") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getKbWrittenForDevice(device), device + "_kB_written") + ); + } + } + + private void registerNetworkInterfaceMetrics() { + NetDevReader reader = new NetDevReader(); + if (!reader.fileExists()) + return; + for (String interfaceName: reader.getInterfaces()) { + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getBytesReceived(interfaceName), interfaceName + "_rx_bytes") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getPacketsReceived(interfaceName), interfaceName + "_rx_packets") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getBytesTransmitted(interfaceName), interfaceName + "_tx_bytes") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getPacketsTransmitted(interfaceName), interfaceName + "_tx_packets") + ); + } + } + + private void registerStatMetrics() { + StatReader reader = new StatReader(); + if (!reader.fileExists()) + return; + Gauge cpuUserGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getUserTime(), "cpu_user") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getSystemTime(), "cpu_system") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getIdleTime(), "cpu_idle") + ); + ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getIoWaitTime(), "cpu_iowait") + ); + Gauge cpuTotalGauge = ActivityMetrics.register( + new NBFunctionGauge(this, () -> reader.getTotalTime(), "cpu_total") + ); + // add checking for percent of time spent in user space; TODO: Modify percent threshold + clientMetricChecker.addRatioMetricToCheck( + "cpu_user_percent", cpuUserGauge, cpuTotalGauge, 50.0, true + ); + } + @Override public NBLabels getLabels() { return labels; diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/ClientSystemMetricChecker.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/ClientSystemMetricChecker.java new file mode 100644 index 000000000..22315690b --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/ClientSystemMetricChecker.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import com.codahale.metrics.Gauge; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.List; + +public class ClientSystemMetricChecker { + private final int pollIntervalSeconds; + private final ScheduledExecutorService scheduler; + private List clientMetrics; + + public ClientSystemMetricChecker(int pollIntervalSeconds) { + this.pollIntervalSeconds = pollIntervalSeconds; + this.scheduler = Executors.newScheduledThreadPool(1); + this.clientMetrics = new ArrayList<>(); + } + + public void addMetricToCheck(String name, Gauge metric, Double threshold) { + addRatioMetricToCheck(name, metric, null, threshold, false); + } + + public void addRatioMetricToCheck(String name, Gauge numerator, Gauge denominator, Double threshold, boolean retainPrev) { + /** + * Some "meaningful" system metrics are derived via: + * - taking a ratio of instantaneous values (e.g. MemUsed / MemTotal from /proc/meminfo) + * - taking a ratio of deltas of aggregates values over a time window (e.g. CPU utilization from /proc/stat) + * + * This method serves to be able to allow checking those which can be derived as a ratio of two existing metrics. + */ + clientMetrics.add(new ClientMetric(name, numerator, denominator, threshold, retainPrev)); + } + + public void start() { + scheduler.scheduleAtFixedRate(() -> { + checkMetrics(); + }, pollIntervalSeconds, pollIntervalSeconds, TimeUnit.SECONDS); + } + + private void checkMetrics() { + for (ClientMetric c: clientMetrics) + c.check(); + } + + public void shutdown() { + scheduler.shutdown(); + } + + private class ClientMetric { + private static final Logger logger = LogManager.getLogger(ClientMetric.class); + private final String name; + private final Gauge numerator; + private final Gauge denominator; + private final Double threshold; + private final Boolean retainPrevValue; + private Double prevNumeratorValue; + private Double prevDenominatorValue; + + private ClientMetric(String name, Gauge gauge, Double threshold) { + this(name, gauge, null, threshold, false); + } + + private ClientMetric(String name, Gauge numerator, Gauge denominator, Double threshold, Boolean retainPrevValue) { + this.name = name; + this.numerator = numerator; + this.denominator = denominator; + this.threshold = threshold; + this.retainPrevValue = retainPrevValue; + this.prevNumeratorValue = null; + this.prevDenominatorValue = null; + } + + private Double extract(){ + Double numeratorVal = numerator.getValue(); + if (numeratorVal == null) + return null; + Double deltaNumeratorVal = numeratorVal; + if (prevNumeratorValue != null) + deltaNumeratorVal -= prevNumeratorValue; + // the case that we are not extracting a ratio of values + if (denominator == null) { + if (retainPrevValue) + prevNumeratorValue = numeratorVal; + return deltaNumeratorVal; + } + // at this point, we will be extracting a ratio of gauge value changes over a time interval + Double denominatorVal = denominator.getValue(); + if (denominatorVal == null) + return null; + Double deltaDenominatorVal = denominatorVal; + if (prevDenominatorValue != null) + deltaDenominatorVal -= prevDenominatorValue; + if (deltaDenominatorVal == 0.0) + return null; + Double percent = (deltaNumeratorVal / deltaDenominatorVal) * 100.0; + if (retainPrevValue) { + prevNumeratorValue = numeratorVal; + prevDenominatorValue = denominatorVal; + } + return percent; + } + + private void check() { + Double extractedVal = extract(); + if (extractedVal != null && extractedVal > threshold) + logger.warn(name + " value = " + extractedVal + " > threshold " + threshold); + } + } +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/DiskStatsReader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/DiskStatsReader.java new file mode 100644 index 000000000..1defe9a8c --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/DiskStatsReader.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.MatchResult; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class DiskStatsReader extends LinuxSystemFileReader { + /** + * Note that all fields are cumulative within /proc/diskstats. + * + * Reference: + * - https://serverfault.com/questions/619097/interpreting-proc-diskstats-for-a-webserver-more-writes-than-reads + * + * Example line: + * 259 0 nvme0n1 669494 21 65326120 388760 3204963 2891102 734524354 42209620 0 446420 41361212 + */ + private static final Double sectorSizeBytes = 512.0; + + public DiskStatsReader() { + super("/proc/diskstats"); + } + + public Double getTransactionsForDevice(String deviceName) { + MatchResult result = findFirstMatch(Pattern.compile(buildRegex(deviceName))); + if (result == null) + return null; + Double readsCompleted = Double.valueOf(result.group(1)); + Double writesCompleted = Double.valueOf(result.group(5)); + return readsCompleted + writesCompleted; + } + + public Double getKbReadForDevice(String deviceName) { + MatchResult result = findFirstMatch(Pattern.compile(buildRegex(deviceName))); + if (result == null) + return null; + Double sectorsRead = Double.valueOf(result.group(3)); + return sectorsRead * sectorSizeBytes / 1024; + } + + public Double getKbWrittenForDevice(String deviceName) { + MatchResult result = findFirstMatch(Pattern.compile(buildRegex(deviceName))); + if (result == null) + return null; + Double sectorsWritten = Double.valueOf(result.group(7)); + return sectorsWritten * sectorSizeBytes / 1024; + } + + private String buildRegex(String deviceName) { + return "\\b" + Pattern.quote(deviceName) + "\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)"; + } + + public List getDevices() { + String regex = "^\\s*\\d+\\s+\\d+\\s+([a-zA-Z0-9]+)\\s+.*$"; + Pattern pattern = Pattern.compile(regex); + List results = findAllLinesMatching(pattern); + return results.stream().map(m -> m.group(1)).collect(Collectors.toList()); + } +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LinuxSystemFileReader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LinuxSystemFileReader.java new file mode 100644 index 000000000..bd9baf12c --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LinuxSystemFileReader.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.Files; +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.MatchResult; +import java.util.regex.Pattern; + +public abstract class LinuxSystemFileReader { + protected Logger logger; + protected String filePath; + + public LinuxSystemFileReader(String filePath) { + logger = LogManager.getLogger(this.getClass()); + this.filePath = filePath; + } + + public boolean fileExists() { + Path path = Paths.get(filePath); + return Files.exists(path); + } + + protected Double extract(String regex, int groupIdx){ + Pattern pattern = Pattern.compile(regex); + MatchResult result = findFirstMatch(pattern); + if (result == null) + return null; + assert (1 <= groupIdx && groupIdx <= result.groupCount()); + return Double.valueOf(result.group(groupIdx)); + } + + protected MatchResult findFirstMatch(Pattern pattern) { + Matcher matcher = null; + try (FileReader file = new FileReader(filePath); + BufferedReader reader = new BufferedReader(file)) { + String line; + while ((line = reader.readLine()) != null) { + matcher = pattern.matcher(line); + if (matcher.find()) + break; + } + } catch (FileNotFoundException e) { + logger.warn("File not found: " + filePath); + } catch (final Throwable t) { + throw new RuntimeException("Failed to read " + filePath); + } + if (matcher == null) + return null; + return matcher.toMatchResult(); + } + + protected List findAllLinesMatching(Pattern pattern) { + List results = new ArrayList<>(); + Matcher matcher; + try (FileReader file = new FileReader(filePath); + BufferedReader reader = new BufferedReader(file)) { + String line; + while ((line = reader.readLine()) != null) { + try { + matcher = pattern.matcher(line); + if (matcher.find()) + results.add(matcher.toMatchResult()); + } catch (Exception e) { + logger.error("Error processing line: " + e.getMessage()); + } + } + } catch (FileNotFoundException e) { + logger.warn("File not found: " + filePath); + } + catch (final Throwable t) { + throw new RuntimeException("Failed to read " + filePath); + } + return results; + } +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LoadAvgReader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LoadAvgReader.java new file mode 100644 index 000000000..c70028610 --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/LoadAvgReader.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class LoadAvgReader extends LinuxSystemFileReader { + /** + * Example line: + * 0.78 1.39 2.03 1/2153 2818 + */ + private static final String regex = "(\\d+\\.\\d+)\\s(\\d+\\.\\d+)\\s(\\d+\\.\\d+)"; + + public LoadAvgReader(){ + super("/proc/loadavg"); + } + + public Double getOneMinLoadAverage() { + return extract(regex, 1); + } + + public Double getFiveMinLoadAverage() { + return extract(regex, 2); + } + + public Double getFifteenMinLoadAverage() { + return extract(regex, 3); + } +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/MemInfoReader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/MemInfoReader.java new file mode 100644 index 000000000..3a33fc060 --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/MemInfoReader.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class MemInfoReader extends LinuxSystemFileReader { + /** + * References: + * - https://docs.kernel.org/filesystems/proc.html#meminfo + * - https://stackoverflow.com/questions/41224738/how-to-calculate-system-memory-usage-from-proc-meminfo-like-htop + */ + public MemInfoReader() { + super("/proc/meminfo"); + } + + public Double getMemTotalkB() { + String regex = "MemTotal:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getMemFreekB() { + String regex = "MemFree:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getMemAvailablekB() { + String regex = "MemAvailable:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getMemUsedkB() { + Double memTotal = getMemTotalkB(); + Double memFree = getMemFreekB(); + if (memTotal != null && memFree != null) + return memTotal - memFree; + return null; + } + + public Double getMemCachedkB() { + String regex = "Cached:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getMemBufferskB() { + String regex = "Buffers:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getSwapTotalkB() { + String regex = "SwapTotal:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getSwapFreekB() { + String regex = "SwapFree:\\s+(\\d+) kB"; + return extract(regex, 1); + } + + public Double getSwapUsedkB() { + Double swapTotal = getSwapTotalkB(); + Double swapFree = getSwapFreekB(); + if (swapTotal != null && swapFree != null) + return swapTotal - swapFree; + return null; + } +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/NetDevReader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/NetDevReader.java new file mode 100644 index 000000000..22bc47378 --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/NetDevReader.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.MatchResult; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class NetDevReader extends LinuxSystemFileReader { + /** + * Note that all fields are cumulative in /proc/net/dev + * + * Reference: + * - https://www.linuxquestions.org/questions/linux-networking-3/need-explanation-of-proc-net-dev-bytes-counters-4175458860/ + * + * Example line: + * wlp59s0: 2941956695 4935327 0 0 0 0 0 0 1213470966 3450551 0 0 0 0 0 0 + */ + public NetDevReader() { + super("/proc/net/dev"); + } + + public Double getBytesReceived(String interfaceName) { + return extract(buildRegex(interfaceName), 1); + } + + public Double getPacketsReceived(String interfaceName) { + return extract(buildRegex(interfaceName), 2); + } + + public Double getBytesTransmitted(String interfaceName) { + return extract(buildRegex(interfaceName), 3); + } + + public Double getPacketsTransmitted(String interfaceName) { + return extract(buildRegex(interfaceName), 4); + } + + private String buildRegex(String interfaceName) { + return "\\b" + Pattern.quote(interfaceName) + "\\s*:(\\s*\\d+)\\s+(\\d+)\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+(\\d+)\\s+(\\d+)"; + } + + public List getInterfaces() { + String regex = "^\\s*([^\\s:]+):\\s*\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+"; + Pattern pattern = Pattern.compile(regex); + List results = findAllLinesMatching(pattern); + return results.stream().map(m -> m.group(1)).collect(Collectors.toList()); + } +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/StatReader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/StatReader.java new file mode 100644 index 000000000..3db080efd --- /dev/null +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/clientload/StatReader.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.core.clientload; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.regex.MatchResult; +import java.util.regex.Pattern; + +public class StatReader extends LinuxSystemFileReader { + /** + * Note that all fields are cumulative within /proc/stat. + * + * Reference: + * - https://docs.kernel.org/filesystems/proc.html#miscellaneous-kernel-statistics-in-proc-stat + * + * Example line: + * cpu 6955150 945 1205506 139439365 115574 0 113356 0 0 0 + */ + private static final String regex = "cpu\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)"; + + public StatReader() { + super("/proc/stat"); + } + + public Double getUserTime() { + return extract(regex, 1); + } + + public Double getSystemTime() { + return extract(regex, 3); + } + + public Double getIdleTime() { + return extract(regex, 4); + } + + public Double getIoWaitTime() { + return extract(regex, 5); + } + + public Double getTotalTime() { + MatchResult result = findFirstMatch(Pattern.compile(regex)); + if (result == null) + return null; + Double user = Double.valueOf(result.group(1)); + Double nice = Double.valueOf(result.group(2)); + Double system = Double.valueOf(result.group(3)); + Double idle = Double.valueOf(result.group(4)); + Double iowait = Double.valueOf(result.group(5)); + Double irq = Double.valueOf(result.group(6)); + Double softirq = Double.valueOf(result.group(7)); + return user + nice + system + idle + iowait + irq + softirq; + } +}