IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

findmax in the house

Browse Source
This commit is contained in:
Jonathan Shook 2023-10-15 23:37:30 -05:00
parent cc8d4be719
commit aaf58984d6
26 changed files with 698 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/ratelimits/simrate/SimRate.java
View File

@ -87,8 +87,8 @@ public class SimRate extends NBBaseComponent implements RateLimiter, Thread.Unca
private void initMetrics() {
create().gauge("cycles_waittime",() -> (double)getWaitTimeDuration().get(ChronoUnit.NANOS));
create().gauge("_config_cyclerate", () -> spec.opsPerSec);
create().gauge("_config_burstrate", () -> spec.burstRatio);
create().gauge("config_cyclerate", () -> spec.opsPerSec);
create().gauge("config_burstrate", () -> spec.burstRatio);
}
public long refill() {
@ -140,10 +140,10 @@ public class SimRate extends NBBaseComponent implements RateLimiter, Thread.Unca
burstFillAllowed = Math.min(this.maxOverActivePool - this.activePool.availablePermits(), burstFillAllowed);
// we can only burst up to our burst limit, but only as much time as we have in the waiting pool already
final int burstFill = (int) Math.min(burstFillAllowed, this.waitingPool.get());
final int burstRecoveryToActivePool = (int) Math.max(0L,Math.min(burstFillAllowed, this.waitingPool.get()));
this.waitingPool.addAndGet(-burstFill);
this.activePool.release(burstFill);
this.waitingPool.addAndGet(-burstRecoveryToActivePool);
this.activePool.release(burstRecoveryToActivePool);
// System.out.print(this);
// System.out.print(ANSI_BrightBlue + " adding=" + allocatedToActivePool);
@ -159,6 +159,7 @@ public class SimRate extends NBBaseComponent implements RateLimiter, Thread.Unca
// return waiting;
} catch (Exception e) {
logger.error(e);
throw new RuntimeException(e);
} finally {
fillerLock.unlock();
long waiting = this.activePool.availablePermits() + this.waitingPool.get();

6
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java
View File

@ -434,6 +434,7 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
}
}
// public synchronized void startActivity() {
// RunStateImage startable = tally.awaitNoneOther(1000L, RunState.Uninitialized, RunState.Stopped);
// if (startable.isTimeout()) {
@ -577,6 +578,11 @@ public class ActivityExecutor implements NBLabeledElement, ParameterMap.Listener
awaitMotorsAtLeastRunning();
}
public boolean awaitAllThreadsOnline(long timeoutMs) {
RunStateImage image = tally.awaitNoneOther(timeoutMs, RunState.Running);
return image.isNoneOther(RunState.Running);
}
private class ThreadsGauge implements Gauge<Double> {
public ThreadsGauge(ActivityExecutor activityExecutor) {
ActivityExecutor ae = activityExecutor;

4
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityRuntimeInfo.java
View File

@ -91,4 +91,8 @@ public class ActivityRuntimeInfo implements ProgressCapable {
public ActivityExecutor getActivityExecutor() {
return executor;
}
public boolean awaitAllThreadsOnline(long timeoutMs) {
return this.executor.awaitAllThreadsOnline(timeoutMs);
}
}

22
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/context/ScenarioActivitiesController.java
View File

@ -93,7 +93,9 @@ public class ScenarioActivitiesController extends NBBaseComponent {
*/
public Activity start(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
return start(ad);
Activity started = start(ad);
awaitAllThreadsOnline(started,30000L);
return started;
}
/**
@ -174,10 +176,27 @@ public class ScenarioActivitiesController extends NBBaseComponent {
runtimeInfo.stopActivity();
}
public boolean awaitAllThreadsOnline(ActivityDef activityDef, long timeoutMs) {
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
if (null == runtimeInfo) {
throw new RuntimeException("could not stop missing activity:" + activityDef);
}
scenariologger.debug("STOP {}", activityDef.getAlias());
return runtimeInfo.awaitAllThreadsOnline(timeoutMs);
}
public synchronized void stop(Activity activity) {
stop(activity.getActivityDef());
}
public boolean awaitAllThreadsOnline(Activity activity, long timeoutMs) {
return awaitAllThreadsOnline(activity.getActivityDef(), timeoutMs);
}
/**
* <p>Stop an activity, given an activity def map. The only part of the map that is important is the
* alias parameter. This method retains the map signature to provide convenience for scripting.</p>
@ -386,6 +405,7 @@ public class ScenarioActivitiesController extends NBBaseComponent {
this.awaitActivity(activityDef, timeoutMs);
}
public boolean awaitActivity(ActivityDef activityDef, long timeoutMs) {
ActivityRuntimeInfo ari = this.activityInfoMap.get(activityDef.getAlias());
if (null == ari) {

26
engine-extensions/src/test/java/io/nosqlbench/optimizers/TestOptimoExperiments.java
View File

@ -32,24 +32,14 @@ import java.util.Random;
/**
*
* Consider JSAT
*
* https://en.wikipedia.org/wiki/Coordinate_descent
*
* http://firsttimeprogrammer.blogspot.com/2014/09/multivariable-gradient-descent.html
*
* https://towardsdatascience.com/machine-learning-bit-by-bit-multivariate-gradient-descent-e198fdd0df85
*
* https://pdfs.semanticscholar.org/d142/3994d7b4462994925663959721130755b275.pdf
*
* file:///tmp/bfgs-example.pdf
*
* https://github.com/vinhkhuc/lbfgs4j
*
* https://github.com/EdwardRaff/JSAT/wiki/Algorithms
*
* http://www.optimization-online.org/DB_FILE/2010/05/2616.pdf
*
* https://github.com/dpressel/sgdtk
* <a href="https://en.wikipedia.org/wiki/Coordinate_descent">...</a>
* <a href="http://firsttimeprogrammer.blogspot.com/2014/09/multivariable-gradient-descent.html>...</a>
* <a href="https://towardsdatascience.com/machine-learning-bit-by-bit-multivariate-gradient-descent-e198fdd0df85>...</a>
* <a href="https://pdfs.semanticscholar.org/d142/3994d7b4462994925663959721130755b275.pdf>...</a>
* <a href="https://github.com/vinhkhuc/lbfgs4j">...</a>
* <a href="https://github.com/EdwardRaff/JSAT/wiki/Algorithms">...</a>
* <a href="http://www.optimization-online.org/DB_FILE/2010/05/2616.pdf">...</a>
* <a href="https://github.com/dpressel/sgdtk">...</a>
*
*/
public class TestOptimoExperiments {

57
nb-api/src/main/java/io/nosqlbench/api/engine/metrics/instruments/NBVariableGauge.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.engine.metrics.instruments;
import io.nosqlbench.api.labels.NBLabeledElement;
import io.nosqlbench.api.labels.NBLabels;
import io.nosqlbench.components.NBComponent;
import java.util.Map;
import java.util.function.DoubleSupplier;
import java.util.function.Supplier;
/**
* Use this gauge type when you are setting the gauge value directly. It is merely a holder
* for the measurement which is injected at the discretion of the scenario.
*/
public class NBVariableGauge implements NBMetricGauge {
private double value;
private final NBLabeledElement parent;
private final NBLabels labels;
public NBVariableGauge(NBComponent parent, String metricFamilyName, double initialValue, String... additionalLabels) {
this.parent = parent;
this.labels = NBLabels.forKV((Object[]) additionalLabels).and("name", metricFamilyName);
this.value = initialValue;
}
@Override
public NBLabels getLabels() {
return labels;
}
@Override
public Double getValue() {
return value;
}
@Override
public String typeName() {
return "gauge";
}
}

8
nb-api/src/main/java/io/nosqlbench/components/NBCreators.java
View File

@ -60,7 +60,7 @@ public class NBCreators {
}
public NBMetricTimer timer(String metricFamilyName) {
return timer(metricFamilyName,4);
return timer(metricFamilyName,3);
}
public NBMetricTimer timer(String metricFamilyName, int hdrdigits) {
NBLabels labels = base.getLabels().and("name", metricFamilyName);
@ -91,6 +91,12 @@ public class NBCreators {
return gauge;
}
public NBVariableGauge variableGauge(String metricFamilyName, double initialValue, String... additionalLabels) {
NBVariableGauge gauge = new NBVariableGauge(base, metricFamilyName, initialValue, additionalLabels);
base.addComponentMetric(gauge);
return gauge;
}
public DoubleSummaryGauge summaryGauge(String name, String... statspecs) {
List<DoubleSummaryGauge.Stat> stats = Arrays.stream(statspecs).map(DoubleSummaryGauge.Stat::valueOf).toList();

47
nbr/src/main/java/io/nosqlbench/scenarios/MultiplicativeValueFuncs.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios;
import java.util.function.DoubleUnaryOperator;
public class MultiplicativeValueFuncs {
/**
* If the value is above the threshold, return the value. If not, return 1.0d;
* @param threshold
* @return
*/
public static DoubleUnaryOperator above(double threshold) {
return (v) -> (v<threshold) ? 1.0d : v;
}
public static DoubleUnaryOperator between(double min, double max) {
return (v) -> (v>=min & v<=max) ? v : 1.0d;
}
public static DoubleUnaryOperator below(double threshold) {
return (v) -> (v>threshold) ? 1.0d : v;
}
public static DoubleUnaryOperator exp_2() {
return (v) -> (v*v)+1;
}
public static DoubleUnaryOperator exp_e() {
return Math::exp;
}
}

2
nbr/src/main/java/io/nosqlbench/scenarios/SC_optimo.java
View File

@ -53,7 +53,7 @@ public class SC_optimo extends SCBaseScenario {
Map<String, String> activityParams = new HashMap<>(Map.of(
"cycles", String.valueOf(Long.MAX_VALUE),
"threads", "1",
"threads", params.getOrDefault("threads","1"),
"driver", "diag",
"rate", "1",
"dryrun","op"

45
nbr/src/main/java/io/nosqlbench/scenarios/ValueFuncs.java
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios;
public class ValueFuncs {
public static double zeroBelow(double value, double threshold) {
if (value<threshold) {
return 0.0d;
}
return value;
}
public static double zeroAbove(double value, double threshold) {
if (value>threshold) {
return 0.0d;
}
return value;
}
/**
* Apply exponential weighting to the value base 2. For rate=1.0, the weight
*/
public static double exp_2(double value) {
return (value*value)+1;
}
public static double exp_e(double value) {
return Math.exp(value);
}
}

36
nbr/src/main/java/io/nosqlbench/scenarios/findmax/Criterion.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios.findmax;
import java.util.function.DoubleSupplier;
import java.util.function.ToDoubleFunction;
public record Criterion(
String name,
EvalType evaltype,
ToDoubleFunction<DoubleMap> remix,
DoubleSupplier supplier,
double weight,
/**
* This frameStartCallback is run at the start of a window
*/
Runnable frameStartCallback
) {
public Criterion {
frameStartCallback = frameStartCallback!=null ? frameStartCallback : () -> {};
}
}

65
nbr/src/main/java/io/nosqlbench/scenarios/findmax/DoubleMap.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios.findmax;
import java.util.Arrays;
public class DoubleMap {
private double[] values = new double[0];
private String[] names = new String[0];
public double put(String name, double value) {
for (int i = 0; i < names.length; i++) {
if (names[i].equals(name)) {
values[i] = value;
return value;
}
}
double[] newValues = new double[values.length + 1];
System.arraycopy(values,0,newValues,0,values.length);
newValues[newValues.length-1]=value;
this.values = newValues;
String[] newNames = new String[names.length + 1];
System.arraycopy(names,0,newNames,0,names.length);
newNames[newNames.length-1]=name;
this.names = newNames;
return value;
}
public double get(String name) {
for (int i = 0; i < names.length; i++) {
if (names[i].equals(name)) {
return values[i];
}
}
throw new RuntimeException("Unknown name '" + name + "': in " + Arrays.toString(names));
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("values: ");
if (values.length>0) {
for (int i = 0; i < values.length; i++) {
sb.append(names[i]).append("=").append(String.format("%.3f",values[i])).append(" ");
}
sb.setLength(sb.length()-1);
}
return sb.toString();
}
}

23
nbr/src/main/java/io/nosqlbench/scenarios/findmax/EvalType.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios.findmax;
public enum EvalType {
direct,
deltaT,
remix
}

2
nbr/src/main/java/io/nosqlbench/scenarios/findmax/FindmaxSearchParams.java
View File

@ -33,7 +33,7 @@ public record FindmaxSearchParams(
) {
public FindmaxSearchParams(ScenarioParams params) {
this(
params.maybeGet("sample_time_ms").map(Integer::parseInt).orElse(1000),
params.maybeGet("sample_time_ms").map(Integer::parseInt).orElse(3000),
params.maybeGet("sample_max").map(Integer::parseInt).orElse(10000),
params.maybeGet("sample_incr").map(Double::parseDouble).orElse(1.01d),
params.maybeGet("rate_base").map(Double::parseDouble).orElse(0d),

92
nbr/src/main/java/io/nosqlbench/scenarios/findmax/FrameSample.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios.findmax;
public record FrameSample(
Criterion criterion,
int index,
long startAt,
long endAt,
double startval,
double endval,
double calculated,
DoubleMap vars
) {
public FrameSample {
vars.put(criterion.name(), calculated);
}
public double weightedValue() {
if (Double.isNaN(criterion.weight())) {
return 1.0d;
} else {
return calculated * criterion().weight();
}
}
private double calculatedValue() {
return switch (criterion.evaltype()) {
case direct -> endval;
case deltaT -> (endval - startval) / seconds();
case remix -> criterion.remix().applyAsDouble(vars);
};
}
private double seconds() {
return ((double) (endAt - startAt)) / 1000d;
}
public static FrameSample init(Criterion criterion, int index, DoubleMap vars) {
return new FrameSample(criterion, index, 0, 0, Double.NaN, Double.NaN, Double.NaN, vars);
}
public FrameSample start(long startTime) {
criterion.frameStartCallback().run();
double v = (criterion().evaltype() == EvalType.deltaT) ? criterion().supplier().getAsDouble() : Double.NaN;
return new FrameSample(criterion, index, startTime, 0L, v, Double.NaN, Double.NaN, vars);
}
public FrameSample stop(long stopTime) {
double v2 = (criterion().evaltype() != EvalType.remix) ? criterion().supplier().getAsDouble() : Double.NaN;
FrameSample intermediate = new FrameSample(criterion, index, startAt, stopTime, startval, v2, Double.NaN, vars);
FrameSample complete = intermediate.tally();
return complete;
}
private FrameSample tally() {
return new FrameSample(criterion, index, startAt, endAt, startval, endval, calculatedValue(), vars);
}
@Override
public String toString() {
return String.format(
"%30s % 3d derived=% 12.3f weighted=% 12.5f%s",
// "%30s %03d dt[%4.2fS] dV[% 10.3f] dC[% 10.3f] wV=%010.5f%s",
criterion.name(),
index,
// seconds(),
// deltaV(),
calculated,
weightedValue(),
(Double.isNaN(criterion.weight()) ? " [NEUTRAL WEIGHT]" : "")
);
}
private double deltaV() {
return endval - startval;
}
}

52
nbr/src/main/java/io/nosqlbench/scenarios/findmax/FrameSampleSet.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios.findmax;
import java.util.ArrayList;
import java.util.List;
public class FrameSampleSet extends ArrayList<FrameSample> {
public FrameSampleSet(List<FrameSample> samples) {
super(samples);
}
public int index() {
return getLast().index();
}
public double value() {
double product = 1.0;
for (FrameSample sample : this) {
double weighted = sample.weightedValue();
product *= weighted;
}
return product;
}
// https://www.w3.org/TR/xml-entity-names/025.html
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(String.format("FRAME % 5d VALUE %10.5f\n", index(), value()));
for (int i = 0; i < this.size(); i++) {
sb.append(i==this.size()-1 ? "┗━▶ ": "┣━▶ ");
sb.append(get(i).toString()).append("\n");
}
return sb.toString();
}
}

4
nbr/src/main/java/io/nosqlbench/scenarios/findmax/JournalView.java
View File

@ -22,4 +22,8 @@ public interface JournalView {
List<SimFrame> frames();
SimFrame last();
SimFrame beforeLast();
SimFrame bestRun();
SimFrame before(SimFrame frame);
SimFrame after(SimFrame frame);
}

55
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SC_findmax.java
View File

@ -16,7 +16,6 @@
package io.nosqlbench.scenarios.findmax;
import io.nosqlbench.api.engine.metrics.instruments.NBFunctionGauge;
import io.nosqlbench.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.api.engine.metrics.instruments.NBMetricTimer;
import io.nosqlbench.components.NBComponent;
@ -57,12 +56,13 @@ public class SC_findmax extends SCBaseScenario {
public void invoke() {
// TODO: having "scenario" here as well as in "named scenario" in workload templates is confusing. Make this clearer.
String workload = params.getOrDefault("workload", "default_workload");
CycleRateSpec ratespec = new CycleRateSpec(100.0, 1.05);
Map<String, String> activityParams = new HashMap<>(Map.of(
"cycles", String.valueOf(Long.MAX_VALUE),
"threads", "1",
"threads", params.getOrDefault("threads","1"),
"driver", "diag",
"rate", "1",
"rate", String.valueOf(ratespec.opsPerSec),
"dryrun", "op"
));
if (params.containsKey("workload")) {
@ -76,33 +76,28 @@ public class SC_findmax extends SCBaseScenario {
FindmaxSearchParams findmaxSettings = new FindmaxSearchParams(params);
int seconds = findmaxSettings.sample_time_ms();
// double target_rate = findmaxSettings.rate_base() + findmaxSettings.rate_step();
int sampletime_ms = findmaxSettings.sample_time_ms();
Activity flywheel = controller.start(activityParams);
final double[] target_rate = new double[] {findmaxSettings.rate_step()};
NBFunctionGauge targetRateGauge = flywheel.create().gauge("target_rate", () -> target_rate[0]);
// stdout.println("warming up for " + seconds + " seconds");
// controller.waitMillis(seconds * 1000);
SimFrameCapture capture = this.perfValueMeasures(flywheel, 0.99, 50);
SimFramePlanner planner = new SimFramePlanner(findmaxSettings);
SimFrameJournal journal = new SimFrameJournal();
SimFrameParams frameParams = planner.initialStep();
while (frameParams!=null) {
stdout.println("params:" + frameParams);
target_rate[0] = frameParams.computed_rate();
flywheel.onEvent(ParamChange.of(new CycleRateSpec(target_rate[0], 1.05d, SimRateSpec.Verb.restart)));
while (frameParams != null) {
stdout.println(frameParams);
flywheel.onEvent(ParamChange.of(new CycleRateSpec(frameParams.computed_rate(), 1.05d, SimRateSpec.Verb.restart)));
capture.startWindow();
controller.waitMillis(frameParams.sample_time_ms());
capture.stopWindow();
journal.record(frameParams,capture.last());
journal.record(frameParams, capture.last());
stdout.println(capture.last());
stdout.println("-".repeat(40));
frameParams = planner.nextStep(journal);
}
controller.stop(flywheel);
stdout.println("bestrun:\n" + journal.bestRun());
// could be a better result if the range is arbitrarily limiting the parameter space.
}
@ -110,22 +105,28 @@ public class SC_findmax extends SCBaseScenario {
private SimFrameCapture perfValueMeasures(Activity activity, double fractional_quantile, double cutoff_ms) {
SimFrameCapture sampler = new SimFrameCapture();
NBMetricTimer result_success_timer = activity.find().timer("name:result_success");
NBMetricTimer result_timer = activity.find().timer("name:result");
NBMetricTimer result_success_timer = activity.find().timer("name:result_success");
NBMetricGauge cyclerate_gauge = activity.find().gauge("name=config_cyclerate");
// achieved rate
sampler.addDeltaTime(
"achieved_rate",
result_success_timer::getCount,
1.0
);
sampler.addDirect("target_rate", cyclerate_gauge::getValue, Double.NaN);
sampler.addDeltaTime("achieved_oprate", result_timer::getCount, Double.NaN);
sampler.addDeltaTime("achieved_ok_oprate", result_success_timer::getCount, 1.0);
// NBMetricGauge target_rate_gauge = activity.find().gauge("name=target_rate");
// sampler.addDirect(
// "achieved_ratio",
// () -> Math.max(1.0d,(result_success_timer.getCount() / target_rate_gauge.getValue()))*Math.max(1.0d,(result_success_timer.getCount() / target_rate_gauge.getValue())),
// 1.0
// );
sampler.addRemix("achieved_success_ratio", vars -> {
// exponentially penalize results which do not attain 100% successful op rate
double basis = Math.min(1.0d, vars.get("achieved_ok_oprate") / vars.get("achieved_oprate"));
return Math.pow(basis,3);
});
sampler.addRemix("achieved_target_ratio", (vars) -> {
// exponentially penalize results which do not attain 100% target rate
double basis = Math.min(1.0d, vars.get("achieved_ok_oprate") / vars.get("target_rate"));
return Math.pow(basis,3);
});
// TODO: add response time with a sigmoid style threshold at fractional_quantile and cutoff_ms
// TODO: add tries based saturation detection, where p99 tries start increasing above 1
// // response time
// sampler.addDirect(

3
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SimFrame.java
View File

@ -21,7 +21,7 @@ package io.nosqlbench.scenarios.findmax;
* @param params The parameters which control the simulated workload during the sample window
* @param result The measured result, including key metrics and criteria for the sample window
*/
public record SimFrame(SimFrameParams params, SimFrameCapture.FrameSampleSet result) {
public record SimFrame(SimFrameParams params, FrameSampleSet result) {
public double value() {
return result().value();
}
@ -29,4 +29,5 @@ public record SimFrame(SimFrameParams params, SimFrameCapture.FrameSampleSet res
return result.index();
}
}

177
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SimFrameCapture.java
View File

@ -21,11 +21,12 @@ import java.util.Collections;
import java.util.List;
import java.util.function.DoubleSupplier;
import java.util.function.LongSupplier;
import java.util.function.ToDoubleFunction;
/**
* This is a helper class that makes it easy to bundle up a combination of measurable
* factors and get a windowed sample from them. To use it, add your named data sources
* with their coefficients, and optionally a callback which resets the measurement
* with their coefficients, and optionally a frameStartCallback which resets the measurement
* buffers for the next time. When you call {@link #getValue()}, all callbacks
* are used after the value computation is complete.
*
@ -37,32 +38,84 @@ public class SimFrameCapture implements SimFrameResults {
private FrameSampleSet currentFrame;
public void addDirect(String name, DoubleSupplier supplier, double weight, Runnable callback) {
this.criteria.add(new Criterion(name, supplier, weight, callback, false));
/**
* Direct values are simply measured at the end of a frame.
*
* @param name
* measure name
* @param supplier
* source of measurement
* @param weight
* coefficient of weight for this measure
* @param callback
*/
private void add(String name, EvalType type, ToDoubleFunction<DoubleMap> remix, DoubleSupplier supplier, double weight, Runnable callback) {
this.criteria.add(new Criterion(name, type, remix, supplier, weight, callback==null? () -> {} : callback));
}
/**
* Direct values are simply measured at the end of a frame.
*
* @param name
* measure name
* @param supplier
* source of measurement
* @param weight
* coefficient of weight for this measure
*/
public void addDirect(String name, DoubleSupplier supplier, double weight) {
addDirect(name, supplier, weight, () -> {
});
add(name, EvalType.direct, null, supplier, weight, null);
}
public void addDeltaTime(String name, DoubleSupplier supplier, double weight, Runnable callback) {
this.criteria.add(new Criterion(name, supplier, weight, callback, true));
this.criteria.add(new Criterion(name, EvalType.deltaT, null, supplier, weight, callback));
}
public void addDeltaTime(String name, DoubleSupplier supplier, double weight) {
addDeltaTime(name, supplier, weight, () -> {
});
criteria.add(new Criterion(name, EvalType.deltaT, null, supplier, weight, null));
}
/**
* Delta Time values are taken as the differential of the first and last values with respect
* to time passing.
*
* @param name
* @param supplier
* @param weight
*/
public void addDeltaTime(String name, LongSupplier supplier, double weight) {
addDeltaTime(name, () -> (double)supplier.getAsLong(), weight);
addDeltaTime(name, () -> (double) supplier.getAsLong(), weight);
}
/**
* A remix function takes as its input the computed raw values of the other functions, irrespective
* of their weights or weighting functions. At the end of a frame, each defined value is computed
* in the order it was added for capture and then added to the results view, where it can be referenced
* by subsequent functions. Thus, any remix values must be added after those value on which it depends.
*
* @param name
* The name of the remix value
* @param remix
* A function which relies on previously computed raw values.
* @param weight
* The weight to apply to the result of this value for the final frame sample value.
* @param callback
* An optional callback to invoke when the frame starts
*/
public void addRemix(String name, ToDoubleFunction<DoubleMap> remix, double weight, Runnable callback) {
add(name, EvalType.remix, remix, null, weight, callback);
}
public void addRemix(String name, ToDoubleFunction<DoubleMap> remix, double weight) {
add(name, EvalType.remix, remix, null, weight, null);
}
@Override
public List<FrameSampleSet> history() {
return Collections.unmodifiableList(this.allFrames);
}
@Override
public double getValue() {
if (allFrames.isEmpty()) {
@ -79,7 +132,7 @@ public class SimFrameCapture implements SimFrameResults {
@Override
public String toString() {
StringBuilder sb = new StringBuilder("PERF VALUE=").append(getValue()).append("\n");
sb.append("windows:\n" + allFrames.getLast().toString());
sb.append("windows:\n").append(allFrames.getLast().toString());
return sb.toString();
}
@ -92,8 +145,19 @@ public class SimFrameCapture implements SimFrameResults {
throw new RuntimeException("cant start window twice in a row. Must close window first");
}
int nextidx = this.allFrames.size();
List<FrameSample> samples = criteria.stream().map(c -> FrameSample.init(c,nextidx).start(now)).toList();
DoubleMap vars = new DoubleMap();
List<FrameSample> samples = criteria.stream().map(c -> FrameSample.init(c, nextidx, vars).start(now)).toList();
this.currentFrame = new FrameSampleSet(samples);
// System.out.println("after start:\n"+ frameCaptureSummary(currentFrame));
}
private String frameCaptureSummary(FrameSampleSet currentFrame) {
StringBuilder sb = new StringBuilder();
for (FrameSample fs : this.currentFrame) {
sb.append(fs.index()).append(" T:").append(fs.startAt()).append("-").append(fs.endAt()).append(" V:")
.append(fs.startval()).append(",").append(fs.endval()).append("\n");
}
return sb.toString();
}
public void stopWindow() {
@ -105,99 +169,20 @@ public class SimFrameCapture implements SimFrameResults {
currentFrame.set(i, currentFrame.get(i).stop(now));
}
allFrames.add(currentFrame);
// System.out.println("after stop:\n"+ frameCaptureSummary(currentFrame));
currentFrame = null;
}
public static record Criterion(
String name,
DoubleSupplier supplier,
double weight,
Runnable callback,
boolean delta
) {
}
public FrameSampleSet last() {
return allFrames.getLast();
}
public void addRemix(String name, ToDoubleFunction<DoubleMap> remix) {
addRemix(name, remix, 1.0, null);
}
public static class FrameSamples extends ArrayList<FrameSampleSet> {
}
public static class FrameSampleSet extends ArrayList<FrameSample> {
public FrameSampleSet(List<FrameSample> samples) {
super(samples);
}
public int index() {
return getLast().index();
}
public double value() {
double product = 1.0;
for (FrameSample sample : this) {
product *= sample.weightedValue();
}
return product;
}
@Override
public String toString() {
StringBuilder sb= new StringBuilder();
sb.append(String.format("FRAME %05d VALUE %010.5f\n", index(), value())).append("\n");
for (FrameSample frameSample : this) {
sb.append(" > ").append(frameSample.toString()).append("\n");
}
return sb.toString();
}
}
public static record FrameSample(Criterion criterion, int index, long startAt, long endAt, double startval, double endval) {
public double weightedValue() {
return rawValue() * criterion().weight;
}
private double rawValue() {
if (criterion.delta()) {
return endval - startval;
}
return endval;
}
private double rate() {
return rawValue() / seconds();
}
private double seconds() {
return ((double) (endAt - startAt)) / 1000d;
}
public static FrameSample init(Criterion criterion, int index) {
return new FrameSample(criterion, index, 0, 0, Double.NaN, Double.NaN);
}
public FrameSample start(long startTime) {
criterion.callback.run();
double v1 = criterion.supplier.getAsDouble();
return new FrameSample(criterion, index, startTime, 0L, v1, Double.NaN);
}
public FrameSample stop(long stopTime) {
double v2 = criterion.supplier.getAsDouble();
return new FrameSample(criterion, index, startAt, stopTime, startval, v2);
}
@Override
public String toString() {
return String.format(
"%20s %03d dt[%04.2f] dV[%010.5f] wV=%010.5f",
criterion.name,
index,
seconds(),
rawValue(),
weightedValue()
);
}
}
}

24
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SimFrameJournal.java
View File

@ -18,10 +18,11 @@ package io.nosqlbench.scenarios.findmax;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
public class SimFrameJournal extends ArrayList<SimFrame> implements JournalView {
public void record(SimFrameParams params, SimFrameCapture.FrameSampleSet samples) {
public void record(SimFrameParams params, FrameSampleSet samples) {
add(new SimFrame(params, samples));
}
@ -42,4 +43,25 @@ public class SimFrameJournal extends ArrayList<SimFrame> implements JournalView
}
return get(size()-2);
}
@Override
public SimFrame bestRun() {
return this.stream().sorted(Comparator.comparingDouble(SimFrame::value)).toList().getLast();
}
@Override
public SimFrame before(SimFrame frame) {
int beforeIdx=frame.index()-1;
if (beforeIdx>=0 && beforeIdx<=size()-1) {
return frames().get(beforeIdx);
} else throw new RuntimeException("Invalid index for before: " + beforeIdx + " with " + size() + " frames");
}
@Override
public SimFrame after(SimFrame frame) {
int afterIdx=frame.index()+1;
if (afterIdx>=0 && afterIdx<=size()-1) {
return frames().get(afterIdx);
} else throw new RuntimeException("Invalid index for after: " + afterIdx + " with " + size() + " frames");
}
}

4
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SimFrameParams.java
View File

@ -17,9 +17,7 @@
package io.nosqlbench.scenarios.findmax;
public record SimFrameParams(
double rate_shelf,
double rate_delta,
long sample_time_ms
double rate_shelf, double rate_delta, long sample_time_ms, String description
) {
public double computed_rate() {
return rate_shelf+rate_delta;

63
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SimFramePlanner.java
View File

@ -19,7 +19,9 @@ package io.nosqlbench.scenarios.findmax;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
public class SimFramePlanner {
private final Logger logger = LogManager.getLogger(SimFramePlanner.class);
@ -49,8 +51,7 @@ public class SimFramePlanner {
public SimFrameParams initialStep() {
return new SimFrameParams(
this.findmax.rate_base(), this.findmax.rate_step(),
this.findmax.sample_time_ms()
this.findmax.rate_base(), this.findmax.rate_step(), this.findmax.sample_time_ms(), "INITIAL"
);
}
@ -64,43 +65,49 @@ public class SimFramePlanner {
* @return Optionally, a set of params which indicates another simulation frame should be sampled, else null
*/
public SimFrameParams nextStep(JournalView journal) {
List<SimFrame> frames = journal.frames();
if (frames.size() < 2) {
System.out.println("FIRSTTWO");
return new SimFrameParams(
journal.last().params().rate_shelf(),
journal.last().params().rate_shelf() + (journal.last().params().rate_delta() * findmax.rate_incr()),
journal.last().params().sample_time_ms()
);
}
SimFrame last = journal.last();
SimFrame before = journal.beforeLast();
if (before.value() < last.value()) { // got a better result, keep on keepin' on
System.out.println("CONTINUE");
SimFrame best = journal.bestRun();
if (best.index() == last.index()) { // got better consecutively
return new SimFrameParams(
last.params().rate_shelf(),
last.params().rate_delta() * findmax.rate_incr(),
last.params().sample_time_ms()
last.params().sample_time_ms(),
"CONTINUE after improvement from frame " + last.index()
);
} else { // reset to last better result as base and start again
if (last.params().rate_delta() > findmax.rate_step()) { // but only if there is still searchable space
System.out.println("REBASE");
return new SimFrameParams(
before.params().computed_rate(),
findmax.rate_step(),
(long) (before.params().sample_time_ms() * findmax.sample_incr()));
} else {
// but only if there is still unsearched resolution within rate_step
} else if (best.index() == last.index() - 1) { // got worse consecutively
if ((last.params().computed_rate() - best.params().computed_rate()) <= findmax.rate_step()) {
logger.info("could not divide search space further, stop condition met");
System.out.println("STOP CONDITION");
return null;
} else {
return new SimFrameParams(
best.params().computed_rate(),
findmax.rate_step(),
(long) (last.params().sample_time_ms() * findmax.sample_incr()),
"REBASE search range to new base after frame " + best.index()
);
}
} else {
// find next frame with higher rate but lower value, the closest one by rate
SimFrame nextWorseFrameWithHigherRate = journal.frames().stream()
.filter(f -> f.value() < best.value())
.filter(f -> f.params().computed_rate() > best.params().computed_rate())
.min(Comparator.comparingDouble(f -> f.params().computed_rate()))
.orElseThrow(() -> new RuntimeException("inconsistent result"));
if ((nextWorseFrameWithHigherRate.params().computed_rate() - best.params().computed_rate()) > findmax.rate_step()) {
return new SimFrameParams(
best.params().computed_rate(),
findmax.rate_step(),
(long) (last.params().sample_time_ms() * findmax.sample_incr()),
"REBASE search range from frames " + best.index() + " ➞ " +nextWorseFrameWithHigherRate.index()
);
} else {
return null;
}
}
}
private SimFrameParams nextStepParams(SimFrame previous) {
return null;
private boolean improvedScore(SimFrame before, SimFrame last) {
return before.value() < last.value();
}
}

2
nbr/src/main/java/io/nosqlbench/scenarios/findmax/SimFrameResults.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.scenarios.findmax;
import java.util.List;
public interface SimFrameResults {
List<SimFrameCapture.FrameSampleSet> history();
List<FrameSampleSet> history();
double getValue();

82
nbr/src/main/java/io/nosqlbench/scenarios/findmax/conditioning/ZScore.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.scenarios.findmax.conditioning;
public class ZScore {
public static void main(String[] args) {
// Example data
double[][] data = {
{1, 2, 3},
{4, 5, 6},
{7, 8, 9}
};
// Normalize data
double[][] normalizedData = zScoreNormalization(data);
// Print normalized data
for (double[] row : normalizedData) {
for (double num : row) {
System.out.print(num + " ");
}
System.out.println();
}
}
public static double[][] zScoreNormalization(double[][] data) {
int numRows = data.length;
int numCols = data[0].length;
double[] means = new double[numCols];
double[] stdDevs = new double[numCols];
// Calculate means
for (double[] row : data) {
for (int j = 0; j < numCols; j++) {
means[j] += row[j];
}
}
for (int j = 0; j < numCols; j++) {
means[j] /= numRows;
}
// Calculate standard deviations
for (double[] row : data) {
for (int j = 0; j < numCols; j++) {
stdDevs[j] += Math.pow(row[j] - means[j], 2);
}
}
for (int j = 0; j < numCols; j++) {
stdDevs[j] = Math.sqrt(stdDevs[j] / numRows);
// Prevent division by zero
if (stdDevs[j] == 0) {
stdDevs[j] = 1;
}
}
// Normalize data
double[][] normalizedData = new double[numRows][numCols];
for (int i = 0; i < numRows; i++) {
for (int j = 0; j < numCols; j++) {
normalizedData[i][j] = (data[i][j] - means[j]) / stdDevs[j];
}
}
return normalizedData;
}
}

15
nbr/src/test/java/io/nosqlbench/scenarios/PerfFrameSamplerTest.java
View File

@ -70,4 +70,19 @@ class PerfFrameSamplerTest {
}
@Test
public void testRemixValues() {
SimFrameCapture pws = new SimFrameCapture();
pws.addDirect("a",() -> 3.0d, 1.0d);
pws.addDirect("b",()-> 7.0d, 1.0d);
pws.addRemix("d", (vars) -> { return vars.get("a")*vars.get("b");},1.0);
pws.startWindow();
pws.stopWindow();
double value = pws.getValue();
// because the value of d is the product of a and b as above
// and the value of the sample is the product of a*b*d
assertThat(value).isCloseTo(21d*21d, Offset.offset(0.002));
}
}