IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

enable direct cycle progress metrics

Browse Source
This commit is contained in:
Jonathan Shook
2023-09-09 00:13:33 -05:00
parent 1137067a19
commit c55213da7c
7 changed files with 126 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/progress/ActivityMetricProgressMeter.java
View File

@@ -16,15 +16,11 @@
package io.nosqlbench.engine.api.activityapi.core.progress;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Timer;
import io.nosqlbench.api.config.NBLabeledElement;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import io.nosqlbench.api.engine.util.Unit;
import io.nosqlbench.engine.api.activityapi.core.Activity;
import java.time.Instant;
import java.util.function.Supplier;
public class ActivityMetricProgressMeter implements ProgressMeterDisplay, CompletedMeter, RemainingMeter, ActiveMeter {
@@ -32,19 +28,12 @@ public class ActivityMetricProgressMeter implements ProgressMeterDisplay, Comple
private final Instant startInstant;
private final Timer bindTimer;
private final Timer cyclesTimer;
private final Gauge<Double> pendingGauge;
private final Gauge<Double> currentGauge;
private final Gauge<Double> completeGauge;
public ActivityMetricProgressMeter(Activity activity) {
this.activity = activity;
this.startInstant = Instant.ofEpochMilli(activity.getStartedAtMillis());
this.bindTimer = activity.getInstrumentation().getOrCreateBindTimer();
this.cyclesTimer = activity.getInstrumentation().getOrCreateCyclesServiceTimer();
this.pendingGauge = ActivityMetrics.gauge(activity,"ops_pending",new AuxGauge(activity, this::getRemainingCount));
this.currentGauge = ActivityMetrics.gauge(activity,"ops_active",new AuxGauge(activity, this::getActiveOps));
this.completeGauge = ActivityMetrics.gauge(activity, "ops_complete", new AuxGauge(activity, this::getCompletedCount));
}
@Override
@@ -86,7 +75,7 @@ public class ActivityMetricProgressMeter implements ProgressMeterDisplay, Comple
@Override
public String toString() {
return getSummary();
return getSummary().toString();
}
@Override
@@ -94,17 +83,5 @@ public class ActivityMetricProgressMeter implements ProgressMeterDisplay, Comple
return cyclesTimer.getCount();
}
private final static class AuxGauge implements Gauge<Double> {
private final NBLabeledElement parent;
private final Supplier<Double> source;
public AuxGauge(NBLabeledElement parent, Supplier<Double> source) {
this.parent = parent;
this.source =source;
}
@Override
public Double getValue() {
return source.get();
}
}
}

6
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/progress/ProgressDisplay.java
View File

@@ -34,14 +34,16 @@ public class ProgressDisplay {
if (meters.length == 0) {
return "";
} else if (meters.length == 1) {
return meters[0].getSummary();
return meters[0].getSummary().toString();
} else {
double total = 0d;
for (ProgressMeterDisplay meter : meters) {
total += meter.getMaxValue();
}
return "PROGRESS:" + ProgressMeterDisplay.format(total / meters.length) + " (" +
Arrays.stream(meters).map(ProgressMeterDisplay::getSummary).collect(Collectors.joining(","));
Arrays.stream(meters).map(ProgressMeterDisplay::getSummary)
.map(Object::toString)
.collect(Collectors.joining(","));
}
}

29
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/progress/ProgressMeterDisplay.java
View File

@@ -48,29 +48,12 @@ public interface ProgressMeterDisplay {
return formatted;
}
default String getSummary() {
StringBuilder legend = new StringBuilder(getProgressName()).append(" (");
StringBuilder values = new StringBuilder("(");
if (this instanceof RemainingMeter remaining) {
legend.append("remaining,");
values.append(String.format("%.0f,",remaining.getRemainingCount()));
}
if (this instanceof ActiveMeter active) {
legend.append("active,");
values.append(String.format("%.0f,",active.getActiveOps()));
}
if (this instanceof CompletedMeter completed) {
legend.append("completed,");
values.append(String.format("%.0f,",completed.getCompletedCount()));
}
legend.setLength(legend.length()-1);
values.setLength(values.length()-1);
// legend.append(" ETA:").append(getETAInstant());
String formatted = legend.append(")=").append(values).append(") ").append(getRatioSummary()).toString();
return formatted;
default ProgressSummary getSummary() {
return new ProgressSummary(
getProgressName(),
(this instanceof RemainingMeter rm) ? rm.getRemainingCount() : -1.0,
(this instanceof ActiveMeter am) ? am.getActiveOps() : -1.0,
(this instanceof CompletedMeter cm) ? cm.getCompletedCount() : -1.0);
}
default long getProgressETAMillis() {

54
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/progress/ProgressSummary.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityapi.core.progress;
import java.util.Locale;
public record ProgressSummary(
String name,
double pending,
double current,
double complete
) {
private String getRatioSummary() {
double progress = complete / (pending+complete);
if (Double.isNaN(progress)) {
return "Unknown";
}
return String.format(Locale.US, "%3.2f%%", (100.0 * progress));
}
public String toString() {
StringBuilder legend = new StringBuilder(name).append(" (");
StringBuilder values = new StringBuilder("(");
legend.append("pending,");
values.append(String.format("%.0f,",pending));
legend.append("current,");
values.append(String.format("%.0f,", current));
legend.append("complete,");
values.append(String.format("%.0f,",complete));
legend.setLength(legend.length()-1);
values.setLength(values.length()-1);
// legend.append(" ETA:").append(getETAInstant());
String formatted = legend.append(")=").append(values).append(") ").append(getRatioSummary()).toString();
return formatted;
}
}

11
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/input/AtomicInput.java
View File

@@ -23,7 +23,6 @@ import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import io.nosqlbench.api.engine.util.Unit;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.core.progress.CycleMeter;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressCapable;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleSegment;
import io.nosqlbench.engine.api.activityapi.input.Input;
@@ -33,6 +32,7 @@ import org.apache.logging.log4j.Logger;
import java.security.InvalidParameterException;
import java.time.Instant;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
/**
* <p>TODO: This documentation is out of date as of 2.0.0
@@ -48,7 +48,7 @@ import java.util.concurrent.atomic.AtomicLong;
* after the max value. They simply expose it to callers. It is up to the
* caller to check the value to determine when the input is deemed "used up."</p>
*/
public class AtomicInput implements Input, ActivityDefObserver, ProgressCapable, Gauge<Long>, NBLabeledElement {
public class AtomicInput implements Input, ActivityDefObserver, Gauge<Long>, NBLabeledElement {
private final static Logger logger = LogManager.getLogger(AtomicInput.class);
private final AtomicLong cycleValue = new AtomicLong(0L);
@@ -156,11 +156,6 @@ public class AtomicInput implements Input, ActivityDefObserver, ProgressCapable,
return true;
}
@Override
public ProgressMeterDisplay getProgressMeter() {
return new AtomicInputProgress(this, activityDef.getAlias(), this);
}
@Override
public NBLabels getLabels() {
return parent.getLabels();
@@ -171,7 +166,7 @@ public class AtomicInput implements Input, ActivityDefObserver, ProgressCapable,
return this.cycleValue.get();
}
private static class AtomicInputProgress implements NBLabeledElement, ProgressMeterDisplay, CycleMeter {
public static class AtomicInputProgress implements NBLabeledElement, ProgressMeterDisplay, CycleMeter {
private final AtomicInput input;
private final String name;

14
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@@ -16,6 +16,7 @@
package io.nosqlbench.engine.api.activityimpl.uniform;
import com.codahale.metrics.Gauge;
import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
@@ -30,6 +31,8 @@ import io.nosqlbench.api.config.NBLabeledElement;
import io.nosqlbench.api.config.NBLabels;
import io.nosqlbench.api.config.standard.*;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.api.engine.metrics.ActivityMetrics;
import io.nosqlbench.api.engine.metrics.instruments.NBFunctionGauge;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
@@ -57,6 +60,10 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
private final ConcurrentHashMap<String, DriverAdapter> adapters = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, OpMapper<Op>> mappers = new ConcurrentHashMap<>();
private final Gauge<Double> pendingOpsGauge;
private final Gauge<Double> activeOpsGauge;
private final Gauge<Double> completeOpsGauge;
public StandardActivity(ActivityDef activityDef, NBLabeledElement parentLabels) {
super(activityDef, parentLabels);
OpsDocList workload;
@@ -144,6 +151,13 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
throw new OpConfigError("Error mapping workload template to operations: " + e.getMessage(), null, e);
}
this.pendingOpsGauge= ActivityMetrics.gauge(this,"ops_pending",
new NBFunctionGauge(this,() -> this.getProgressMeter().getSummary().pending()));
this.activeOpsGauge = ActivityMetrics.gauge(this,"ops_active",
new NBFunctionGauge(this,() -> this.getProgressMeter().getSummary().current()));
this.completeOpsGauge= ActivityMetrics.gauge(this,"ops_complete",
new NBFunctionGauge(this,() -> this.getProgressMeter().getSummary().complete()));
}
@Override

44
nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBFunctionGauge.java Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.engine.metrics.instruments;
import io.nosqlbench.api.config.NBLabeledElement;
import io.nosqlbench.api.config.NBLabels;
import java.util.function.Supplier;
public class NBFunctionGauge implements NBMetricGauge<Double> {
private final Supplier<Double> source;
private final NBLabeledElement parent;
private final NBLabels labels;
public NBFunctionGauge(NBLabeledElement parent, Supplier<Double> source, Object... labels) {
this.parent = parent;
this.labels = NBLabels.forKV(labels);
this.source = source;
}
@Override
public Double getValue() {
return source.get();
}
@Override
public NBLabels getLabels() {
return parent.getLabels().and(this.labels);
}
}