IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint

Browse Source
This commit is contained in:
Jonathan Shook
2023-10-08 13:58:07 -05:00
parent 0699b446a6
commit 9fa711b7ab
71 changed files with 776 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nb-api/src/main/java/io/nosqlbench/api/config/standard/TestComponent.java
View File

@@ -39,7 +39,7 @@ public class TestComponent extends NBBaseComponent {
@Override
public String toString() {
return description();
return "TestComponent #"+hashCode();
}
@Override
@@ -64,4 +64,5 @@ public class TestComponent extends NBBaseComponent {
public void beforeDetach() {
// logger.debug("before detach " + description());
}
}

2
nb-api/src/main/java/io/nosqlbench/api/engine/metrics/reporters/CsvReporter.java
View File

@@ -75,7 +75,7 @@ public class CsvReporter extends PeriodicTaskComponent {
}
public void start() {
List<NBMetric> metrics = component.findMetricsInTree("");
List<NBMetric> metrics = component.find().metrics();
final long timestamp = TimeUnit.MILLISECONDS.toSeconds(clock.getTime());
for (NBMetric metric : metrics) {
if (metric instanceof Gauge<?>) {

2
nb-api/src/main/java/io/nosqlbench/api/engine/metrics/reporters/PromPushReporterComponent.java
View File

@@ -110,7 +110,7 @@ public class PromPushReporterComponent extends PeriodicTaskComponent implements
StringBuilder sb = new StringBuilder(1024 * 1024); // 1M pre-allocated to reduce heap churn
int total = 0;
for (final Object metric : getParent().findMetricsInTree("")) {
for (final Object metric : getParent().find().metrics()) {
sb = PromExpositionFormat.format(nowclock, sb, metric);
total++;
}

2
nb-api/src/main/java/io/nosqlbench/components/AttachedMetricsPushReporter.java
View File

@@ -123,7 +123,7 @@ public class AttachedMetricsPushReporter extends NBBaseComponent implements NBCo
StringBuilder sb = new StringBuilder(1024 * 1024); // 1M pre-allocated to reduce heap churn
List<NBMetric> metrics = new ArrayList<>();
Iterator<NBComponent> allMetrics = NBComponentTraversal.traverseBreadth(getParent());
allMetrics.forEachRemaining(m -> metrics.addAll(m.findMetrics("")));
allMetrics.forEachRemaining(m -> metrics.addAll(m.findComponentMetrics("")));
int total = 0;
for (NBMetric metric : metrics) {

2
nb-api/src/main/java/io/nosqlbench/components/AttachedMetricsSummaryReporter.java
View File

@@ -42,7 +42,7 @@ public class AttachedMetricsSummaryReporter extends PeriodicTaskComponent {
StringBuilder sb = new StringBuilder(1024 * 1024); // 1M pre-allocated to reduce heap churn
List<NBMetric> metrics = new ArrayList<>();
Iterator<NBComponent> allMetrics = NBComponentTraversal.traverseBreadth(getParent());
allMetrics.forEachRemaining(m -> metrics.addAll(m.findMetrics("")));
allMetrics.forEachRemaining(m -> metrics.addAll(m.findComponentMetrics("")));
int total = 0;
for (NBMetric metric : metrics) {

45
nb-api/src/main/java/io/nosqlbench/components/NBBaseComponent.java
View File

@@ -82,38 +82,6 @@ public class NBBaseComponent extends NBBaseComponentMetrics implements NBCompone
return effectiveLabels;
}
@Override
public NBMetric lookupMetricInTree(String name) {
Iterator<NBComponent> tree = NBComponentTraversal.traverseBreadth(this);
while (tree.hasNext()) {
NBComponent c = tree.next();
NBMetric metric = c.lookupMetric(name);
if (metric != null) return metric;
}
return null;
}
@Override
public List<NBMetric> findMetricsInTree(String pattern) {
Iterator<NBComponent> tree = NBComponentTraversal.traverseBreadth(this);
List<NBMetric> found = new ArrayList<>();
while (tree.hasNext()) {
NBComponent c = tree.next();
found.addAll(c.findMetrics(pattern));
}
return found;
}
@Override
public NBMetric findOneMetricInTree(String pattern) {
List<NBMetric> found = findMetricsInTree(pattern);
if (found.size() != 1) {
System.out.println("Runtime Components and Metrics at this time:\n" + NBComponentFormats.formatAsTree(this));
throw new RuntimeException("Found " + found.size() + " metrics with pattern '" + pattern + "', expected exactly 1");
}
return found.get(0);
}
@Override
public void beforeDetach() {
@@ -156,4 +124,17 @@ public class NBBaseComponent extends NBBaseComponentMetrics implements NBCompone
public NBFinders find() {
return new NBFinders(this);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(getClass().getSimpleName());
if (getComponentMetrics().size()>0) {
sb.append(System.lineSeparator()).append("metrics:");
for (NBMetric componentMetric : getComponentMetrics()) {
sb.append(System.lineSeparator()).append("m ").append(componentMetric.toString());
}
}
return sb.toString();
}
}

15
nb-api/src/main/java/io/nosqlbench/components/NBBaseComponentMetrics.java
View File

@@ -19,17 +19,19 @@ package io.nosqlbench.components;
import io.nosqlbench.adapters.api.util.TagFilter;
import io.nosqlbench.api.engine.metrics.instruments.NBMetric;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public class NBBaseComponentMetrics implements NBComponentMetrics {
private final Lock lock = new ReentrantLock(false);
private final Map<String, NBMetric> metrics = new HashMap<>();
private final Map<String, NBMetric> metrics = new ConcurrentHashMap<>();
@Override
public String addMetric(NBMetric metric) {
public String addComponentMetric(NBMetric metric) {
try {
lock.lock();
String openMetricsName = metric.getLabels().linearizeAsMetrics();
@@ -43,13 +45,18 @@ public class NBBaseComponentMetrics implements NBComponentMetrics {
}
}
@Override
public NBMetric lookupMetric(String name) {
public NBMetric getComponentMetric(String name) {
return metrics.get(name);
}
@Override
public List<NBMetric> findMetrics(String pattern) {
public List<NBMetric> findComponentMetrics(String pattern) {
TagFilter filter = new TagFilter(pattern);
return filter.filterLabeled(metrics.values());
}
@Override
public Collection<? extends NBMetric> getComponentMetrics() {
return metrics.values();
}
}

12
nb-api/src/main/java/io/nosqlbench/components/NBBuilders.java
View File

@@ -60,14 +60,14 @@ public class NBBuilders {
public NBMetricTimer timer(String metricFamilyName, int hdrdigits) {
NBLabels labels = base.getLabels().and("name", metricFamilyName);
NBMetricTimer timer = new NBMetricTimer(labels, new DeltaHdrHistogramReservoir(labels, hdrdigits));
base.addMetric(timer);
base.addComponentMetric(timer);
return timer;
}
public Meter meter(String metricFamilyName) {
NBLabels labels = base.getLabels().and("name", metricFamilyName);
NBMetricMeter meter = new NBMetricMeter(labels);
base.addMetric(meter);
base.addComponentMetric(meter);
return meter;
}
@@ -75,14 +75,14 @@ public class NBBuilders {
public NBMetricCounter counter(String metricFamilyName) {
NBLabels labels = base.getLabels().and("name", metricFamilyName);
NBMetricCounter counter = new NBMetricCounter(labels);
base.addMetric(counter);
base.addComponentMetric(counter);
return counter;
}
public NBFunctionGauge gauge(String metricFamilyName, Supplier<Double> valueSource) {
NBFunctionGauge gauge = new NBFunctionGauge(base, valueSource, metricFamilyName);
base.addMetric(gauge);
base.addComponentMetric(gauge);
return gauge;
}
@@ -93,7 +93,7 @@ public class NBBuilders {
DoubleSummaryGauge anyGauge = null;
for (DoubleSummaryGauge.Stat stat : stats) {
anyGauge = new DoubleSummaryGauge(base.getLabels().and(NBLabels.forKV("name",name,"stat", stat)), stat, reservoir);
base.addMetric(anyGauge);
base.addComponentMetric(anyGauge);
}
return anyGauge;
}
@@ -104,7 +104,7 @@ public class NBBuilders {
public NBMetricHistogram histogram(String metricFamilyName, int hdrdigits) {
NBLabels labels = base.getLabels().and("name", metricFamilyName);
NBMetricHistogram histogram = new NBMetricHistogram(labels, new DeltaHdrHistogramReservoir(labels, hdrdigits));
base.addMetric(histogram);
base.addComponentMetric(histogram);
return histogram;
}

2
nb-api/src/main/java/io/nosqlbench/components/NBComponent.java
View File

@@ -33,7 +33,7 @@ import java.util.List;
*
* This interface includes more aspects of above by extension going forward.
*/
public interface NBComponent extends AutoCloseable, NBLabeledElement, NBComponentMetrics, NBMetricsQuery, NBComponentServices {
public interface NBComponent extends AutoCloseable, NBLabeledElement, NBComponentMetrics, NBComponentServices {
NBComponent EMPTY_COMPONENT = new NBBaseComponent(null);

11
nb-api/src/main/java/io/nosqlbench/components/NBComponentFormats.java
View File

@@ -26,7 +26,7 @@ public class NBComponentFormats {
return sb.toString();
}
private final static class PrintVisitor implements NBComponentTraversal.Visitor {
protected final static class PrintVisitor implements NBComponentTraversal.Visitor {
private final StringBuilder builder;
@@ -36,7 +36,14 @@ public class NBComponentFormats {
@Override
public void visit(NBComponent component, int depth) {
builder.append(String.format("%03d %s\n",depth,component));
String indent = " ".repeat(depth);
builder.append(indent).append(String.format("%03d %s",depth,component.description()));
String string = component.toString();
String[] split = string.split(System.lineSeparator());
for (String s : split) {
builder.append(System.lineSeparator()).append(indent).append(" >").append(s);
}
builder.append(System.lineSeparator());
}
}

23
nb-api/src/main/java/io/nosqlbench/components/NBComponentMetrics.java
View File

@@ -18,11 +18,19 @@ package io.nosqlbench.components;
import io.nosqlbench.api.engine.metrics.instruments.NBMetric;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* This is the stateful store of metrics on a specific component in the hierarchy.
* Mostly, these methods provide the internal logic needed to support easier
* access to metrics via {@link NBComponentServices}.
*
*
*/
public interface NBComponentMetrics {
String addMetric(NBMetric metric);
String addComponentMetric(NBMetric metric);
/**
* If you have the serialized open metrics name of a metric, you can ask for it
@@ -30,20 +38,17 @@ public interface NBComponentMetrics {
* @param name The name of a metric in {@code {a:"b",...}} form
* @return the metric or null if it dosen't exist
*/
NBMetric lookupMetric(String name);
NBMetric getComponentMetric(String name);
default Optional<NBMetric> lookupMetricOptionally(String name) {
return Optional.ofNullable(lookupMetric(name));
}
List<NBMetric> findComponentMetrics(String pattern);
List<NBMetric> findMetrics(String pattern);
default NBMetric findOneMetric(String pattern) {
List<NBMetric> found = findMetrics(pattern);
default NBMetric findOneComponentMetric(String pattern) {
List<NBMetric> found = findComponentMetrics(pattern);
if (found.size()!=1) {
throw new RuntimeException("Found " + found.size() + " metrics with pattern '" + pattern + "', expected exactly 1");
}
return found.get(0);
}
Collection<? extends NBMetric> getComponentMetrics();
}

82
nb-api/src/main/java/io/nosqlbench/components/NBFinders.java
View File

@@ -16,9 +16,11 @@
package io.nosqlbench.components;
import io.nosqlbench.api.engine.metrics.instruments.NBMetric;
import io.nosqlbench.api.engine.metrics.instruments.NBMetricCounter;
import io.nosqlbench.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.api.engine.metrics.instruments.*;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
public class NBFinders {
private final NBBaseComponent base;
@@ -28,12 +30,16 @@ public class NBFinders {
}
public NBMetric metric(String pattern) {
NBMetric metric = base.lookupMetricInTree(pattern);
if (metric!=null) { return metric; };
metric = base.findOneMetricInTree(pattern);
return metric;
return oneMetricInTree(pattern);
}
private <T extends NBMetric> T findOneMetricWithType(String pattern, Class<T> clazz) {
public List<NBMetric> metrics(String pattern) {
return this.metricsInTree(pattern);
}
public List<NBMetric> metrics() {
return this.metricsInTree();
}
private <T extends NBMetric> T oneMetricWithType(String pattern, Class<T> clazz) {
NBMetric found = metric(pattern);
if (found==null) {
System.out.println(NBComponentFormats.formatAsTree(base));
@@ -52,13 +58,65 @@ public class NBFinders {
}
public NBMetricGauge metricGauge(String pattern) {
return findOneMetricWithType(pattern, NBMetricGauge.class);
public NBMetricGauge gauge(String pattern) {
return oneMetricWithType(pattern, NBMetricGauge.class);
}
public NBMetricCounter metricCounter(String pattern) {
return findOneMetricWithType(pattern, NBMetricCounter.class);
public NBMetricCounter counter(String pattern) {
return oneMetricWithType(pattern, NBMetricCounter.class);
}
public NBMetricTimer timer(String pattern) {
return oneMetricWithType(pattern, NBMetricTimer.class);
}
public NBMetricHistogram histogram(String pattern) {
return oneMetricWithType(pattern, NBMetricHistogram.class);
}
public NBMetricMeter meter(String pattern) {
return oneMetricWithType(pattern,NBMetricMeter.class);
}
// public NBMetric firstMetricInTree(String name) {
// Iterator<NBComponent> tree = NBComponentTraversal.traverseBreadth(base);
// while (tree.hasNext()) {
// NBComponent c = tree.next();
// NBMetric metric = base.getComponentMetric(name);
// if (metric != null) return metric;
// }
// return null;
// }
private List<NBMetric> metricsInTree() {
Iterator<NBComponent> tree = NBComponentTraversal.traverseBreadth(base);
List<NBMetric> found = new ArrayList<>();
while (tree.hasNext()) {
NBComponent c = tree.next();
found.addAll(c.getComponentMetrics());
}
return found;
}
private List<NBMetric> metricsInTree(String pattern) {
if (pattern.isEmpty()) {
throw new RuntimeException("non-empty predicate is required for this form. Perhaps you wanted metricsInTree()");
}
Iterator<NBComponent> tree = NBComponentTraversal.traverseBreadth(base);
List<NBMetric> found = new ArrayList<>();
while (tree.hasNext()) {
NBComponent c = tree.next();
found.addAll(c.findComponentMetrics(pattern));
}
return found;
}
private NBMetric oneMetricInTree(String pattern) {
List<NBMetric> found = metricsInTree(pattern);
if (found.size() != 1) {
System.out.println("Runtime Components and Metrics at this time:\n" + NBComponentFormats.formatAsTree(base));
throw new RuntimeException("Found " + found.size() + " metrics with pattern '" + pattern + "', expected exactly 1");
}
return found.get(0);
}
}

42
nb-api/src/main/java/io/nosqlbench/components/NBMetricsQuery.java
View File

@@ -1,42 +0,0 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.components;
import io.nosqlbench.api.engine.metrics.instruments.NBMetric;
import java.util.List;
import java.util.Optional;
public interface NBMetricsQuery {
/**
* If you have the serialized open metrics name of a metric, you can ask for it
* this way and get a direct result.
* @param name The name of a metric in {@code {a:"b",...}} form
* @return the metric or null if it dosen't exist
*/
NBMetric lookupMetricInTree(String name);
default Optional<NBMetric> lookupMetricOptionallyInTree(String name) {
return Optional.ofNullable(lookupMetricInTree(name));
}
List<NBMetric> findMetricsInTree(String pattern);
NBMetric findOneMetricInTree(String pattern);
}

4
nb-api/src/main/java/io/nosqlbench/engine/api/scripting/DiagWriter.java
View File

@@ -39,8 +39,8 @@ public class DiagWriter extends PrintWriter {
Writer wrapped;
InterjectingCharArrayWriter buffer;
public DiagWriter(Writer wrapped, InterjectingCharArrayWriter buffer) {
super(new FanWriter(buffer,wrapped));
public DiagWriter(Writer... writers) {
super(new FanWriter(writers));
this.wrapped = wrapped;
this.buffer = buffer;
}

35
nb-api/src/test/java/io/nosqlbench/api/config/standard/TestComponentViewTest.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.config.standard;
import io.nosqlbench.components.NBComponent;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class TestComponentViewTest {
@Test
public void testDiagnosticView() {
NBComponent root = new TestComponent("rootk","rootv");
TestComponent tc = new TestComponent(root, "atest", "view");
String string = tc.toString();
assertThat(string).isEqualTo("TestComponent #1305486145");
}
}

16
nb-api/src/test/java/io/nosqlbench/components/NBBaseComponentMetricsTest.java
View File

@@ -30,22 +30,22 @@ class NBBaseComponentMetricsTest {
void testBasicAddAndLookup() {
NBBaseComponentMetrics cm = new NBBaseComponentMetrics();
NBMetric m1 = new NBBaseMetric("k","20");
String m1Handle = cm.addMetric(m1);
String m1Handle = cm.addComponentMetric(m1);
NBMetric m2 = new NBBaseMetric("k","27","l","62");
String m2Handle = cm.addMetric(m2);
String m2Handle = cm.addComponentMetric(m2);
assertThat(cm.lookupMetric(m1Handle)).isEqualTo(m1);
assertThat(cm.lookupMetric(m2Handle)).isEqualTo(m2);
assertThat(cm.getComponentMetric(m1Handle)).isEqualTo(m1);
assertThat(cm.getComponentMetric(m2Handle)).isEqualTo(m2);
}
@Test
void find() {
NBBaseComponentMetrics cm = new NBBaseComponentMetrics();
NBMetric m1 = new NBBaseMetric("k","20");
String m1Handle = cm.addMetric(m1);
String m1Handle = cm.addComponentMetric(m1);
NBMetric m2 = new NBBaseMetric("k","27","l","62");
String m2Handle = cm.addMetric(m2);
String m2Handle = cm.addComponentMetric(m2);
assertThat(cm.findMetrics("k=27")).isEqualTo(List.of(m2));
assertThat(cm.findMetrics("k=20")).isNotEqualTo(List.of(m2));
assertThat(cm.findComponentMetrics("k=27")).isEqualTo(List.of(m2));
assertThat(cm.findComponentMetrics("k=20")).isNotEqualTo(List.of(m2));
}
}

47
nb-api/src/test/java/io/nosqlbench/components/NBComponentFormatsTest.java Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.components;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.*;
import io.nosqlbench.api.config.standard.TestComponent;
import org.junit.jupiter.api.Test;
public class NBComponentFormatsTest {
@Test
public void testFormat() {
NBComponent root = new TestComponent("rootk","rootv");
TestComponent tc = new TestComponent(root, "atest", "view");
StringBuilder sb = new StringBuilder();
NBComponentFormats.PrintVisitor visitor = new NBComponentFormats.PrintVisitor(sb);
visitor.visit(root,2);
visitor.visit(tc,3);
String diagview = sb.toString();
diagview=diagview.replaceAll("#[0-9]+","#id");
assertThat(diagview).isEqualTo("""
002 TestComponent {rootk="rootv"}
>TestComponent #id
003 TestComponent {atest="view",rootk="rootv"}
>TestComponent #id
""");
}
}

2
nb-api/src/test/java/io/nosqlbench/components/NBComponentServicesTest.java
View File

@@ -48,7 +48,7 @@ class NBComponentServicesTest {
NBFunctionGauge gauge = b1.create().gauge("test_gauge", () -> 5.2d);
String gaugeHandle = gauge.getHandle();
List<NBMetric> metricsInTree = root.findMetricsInTree("");
List<NBMetric> metricsInTree = root.find().metrics();
assertThat(metricsInTree).containsAll(List.of(timer1, gauge));
metricsInTree.forEach(m -> {
System.out.println("metric: " + m.toString());

14
nb-api/src/test/java/io/nosqlbench/components/NBMetricsQueryTest.java
View File

@@ -31,24 +31,24 @@ class NBMetricsQueryTest {
private final static TestComponent root_c2 = new TestComponent(root,"c2","c2");
private final static TestComponent root_c3 = new TestComponent(root,"c3","c3");
private final static NBMetric m1 = new NBBaseMetric("m1","m1");
private final static String m1Handle = root.addMetric(m1);
private final static String m1Handle = root.addComponentMetric(m1);
private final static NBMetric m2 = new NBBaseMetric("m2","m2");
private final static String m2Handle = root_c2.addMetric(m2);
private final static String m2Handle = root_c2.addComponentMetric(m2);
private final static NBMetric m3 = new NBBaseMetric("m3","m3");
private final static String m3Handle = root_c3.addMetric(m3);
private final static String m3Handle = root_c3.addComponentMetric(m3);
@Test
public void testFindInTree() {
NBMetric expectedM3 = root.findOneMetricInTree("m3:m3");
NBMetric expectedM3 = root.find().metric("m3:m3");
assertThat(expectedM3).isEqualTo(m3);
assertThatThrownBy(() -> root.findOneMetricInTree("m3:m4")).isOfAnyClassIn(RuntimeException.class);
assertThatThrownBy(() -> root.find().metric("m3:m4")).isOfAnyClassIn(RuntimeException.class);
}
@Test
public void testFindOneInTree() {
List<NBMetric> metricsInTree = root.findMetricsInTree("");
List<NBMetric> metricsInTree = root.find().metrics();
assertThat(metricsInTree).containsExactly(m1, m2, m3);
List<NBMetric> m3Only = root.findMetricsInTree("m3:m3");
List<NBMetric> m3Only = root.find().metrics("m3:m3");
assertThat(m3Only).containsExactly(m3);
}
}

2
nb-api/src/test/resources/log4j2.xml
View File

@@ -15,7 +15,7 @@
~ limitations under the License.
-->
<Configuration status="debug" strict="true" name="XMLConfigTest">
<Configuration status="warn" strict="true" name="XMLConfigTest">
<Filter type="ThresholdFilter" level="trace"/>