From ff079c04bc16882d4c995a221c3d21e6ae7c9ba2 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Mon, 18 Mar 2024 18:23:41 -0400 Subject: [PATCH] EOD checkin - not ready for use --- .../optimizers/findmax/CMD_findmax.java | 134 +++++------------- ...SearchSettings.java => FindmaxConfig.java} | 10 +- .../findmax/FindmaxFrameFunction.java | 16 ++- 3 files changed, 51 insertions(+), 109 deletions(-) rename nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/{FindmaxSearchSettings.java => FindmaxConfig.java} (87%) diff --git a/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/CMD_findmax.java b/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/CMD_findmax.java index 8c3918bdb..2d816e801 100644 --- a/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/CMD_findmax.java +++ b/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/CMD_findmax.java @@ -24,30 +24,21 @@ import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContaine import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBBaseCommand; import io.nosqlbench.nb.api.components.events.ParamChange; -import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge; -import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram; -import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer; import io.nosqlbench.scenarios.simframe.SimFrameUtils; import io.nosqlbench.scenarios.simframe.capture.SimFrameCapture; import io.nosqlbench.scenarios.simframe.capture.SimFrameJournal; +import io.nosqlbench.scenarios.simframe.capture.SimFrameValueData; import io.nosqlbench.scenarios.simframe.optimizers.CMD_optimize; import io.nosqlbench.scenarios.simframe.planning.SimFrame; import io.nosqlbench.scenarios.simframe.planning.SimFrameFunction; -import io.nosqlbench.scenarios.simframe.stabilization.StatFunctions; import org.apache.commons.math4.legacy.exception.MathIllegalStateException; -import org.apache.commons.math4.legacy.optim.InitialGuess; -import org.apache.commons.math4.legacy.optim.MaxEval; import org.apache.commons.math4.legacy.optim.OptimizationData; import org.apache.commons.math4.legacy.optim.PointValuePair; -import org.apache.commons.math4.legacy.optim.nonlinear.scalar.GoalType; -import org.apache.commons.math4.legacy.optim.nonlinear.scalar.ObjectiveFunction; -import org.apache.commons.math4.legacy.optim.nonlinear.scalar.noderiv.BOBYQAOptimizer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.PrintWriter; import java.io.Reader; -import java.util.List; public class CMD_findmax extends NBBaseCommand { private final static Logger logger = LogManager.getLogger(CMD_optimize.class); @@ -62,108 +53,55 @@ public class CMD_findmax extends NBBaseCommand { stdout.println("starting analysis on activity '" + flywheel.getAlias() + "'"); SimFrameUtils.awaitActivity(flywheel); - /* - var frameParams = initialStep(); - - while (frameParams != null) { - stdout.println(frameParams); - applyParams(frameParams,flywheel); - capture.startWindow(); - if (this instanceof HoldAndSample has) { - has.holdAndSample(capture); - } else { - capture.awaitSteadyState(); - } - capture.stopWindow(); - journal.record(frameParams, capture.last()); - stdout.println(capture.last()); - stdout.println("-".repeat(40)); - frameParams = nextStep(journal); - } - return journal.bestRun().params(); - - */ SimFrameJournal journal = new SimFrameJournal<>(); FindmaxParamModel model = new FindmaxParamModel(); - FindmaxSearchSettings findmaxSearchParams = new FindmaxSearchSettings(params, model); + FindmaxConfig findmaxConfig = new FindmaxConfig(params); -// initial step: -// new io.nosqlbench.scenarios.simframe.optimizers.planners.findmax.FindmaxFrameParams( -// findmaxSearchParams.rate_base(), -// findmaxSearchParams.rate_step(), -// findmaxSearchParams.sample_time_ms(), -// findmaxSearchParams.min_settling_ms(), -// "INITIAL" -// ); - - model.add("rate", 10, findmaxSearchParams.rate_base(), findmaxSearchParams.rate_base()*10, - rate -> flywheel.onEvent(ParamChange.of(new CycleRateSpec(rate, 1.1d, SimRateSpec.Verb.restart))) - //rate -> flywheel.onEvent(ParamChange.of(new CycleRateSpec(params.rate_shelf()+params.rate_delta(), 1.1d, SimRateSpec.Verb.restart))); + model.add("rate", + findmaxConfig.rate_base(), // min + findmaxConfig.rate_base(), // initial + findmaxConfig.sample_max(), // max + rate -> flywheel.onEvent(ParamChange.of(new CycleRateSpec( + findmaxConfig.rate_base() + findmaxConfig.rate_step(), + 1.1d, + SimRateSpec.Verb.restart))) ); - SimFrameCapture capture = this.perfValueMeasures(flywheel, findmaxSearchParams); - SimFrameFunction frameFunction = new FindmaxFrameFunction(controller, findmaxSearchParams, flywheel, capture, journal); - - List od = List.of( - new ObjectiveFunction(frameFunction), - GoalType.MAXIMIZE, - new InitialGuess(model.getInitialGuess()), - new MaxEval(100), - model.getBounds() - ); - - BOBYQAOptimizer mo = new BOBYQAOptimizer( - 6, - 25, - 1E-4 - ); - PointValuePair result = null; - try { - result = mo.optimize(od.toArray(new OptimizationData[0])); - } catch (MathIllegalStateException missed) { - if (missed.getMessage().contains("trust region step has failed to reduce Q")) { - logger.warn(missed.getMessage() + ", so returning current result."); - result = new PointValuePair(journal.last().params().paramValues(), journal.last().value()); - } else { - throw missed; - } - } + SimFrameCapture capture = new SimFrameValueData(flywheel); + SimFrameFunction frameFunction = new FindmaxFrameFunction(controller, findmaxConfig, flywheel, capture, journal, model); + FindmaxAnalyzer analyzer = new FindmaxAnalyzer(frameFunction, model); + FindmaxFrameParams result = analyzer.analyze(); stdout.println("result:" + result); SimFrame best = journal.bestRun(); stdout.println("bestrun:\n" + best); return best.params(); - // could be a better result if the range is arbitrarily limiting the parameter space. } - private SimFrameCapture perfValueMeasures(Activity activity, FindmaxSearchSettings settings) { - SimFrameCapture sampler = new SimFrameCapture(); + private class FindmaxAnalyzer { + private final SimFrameFunction frameFunction; + private final FindmaxParamModel model; + public FindmaxAnalyzer(SimFrameFunction frameFunction, FindmaxParamModel model) { + this.frameFunction = frameFunction; + this.model = model; + } - NBMetricTimer result_timer = activity.find().timer("name:result"); - NBMetricTimer latency_histo = result_timer.attachHdrDeltaHistogram(); + public FindmaxFrameParams analyze() { + FindmaxFrameParams frameParams = new FindmaxFrameParams(model, null); - NBMetricTimer result_success_timer = activity.find().timer("name:result_success"); - NBMetricGauge cyclerate_gauge = activity.find().gauge("name=config_cyclerate"); - NBMetricHistogram tries_histo_src = activity.find().histogram("name=tries"); - NBMetricHistogram tries_histo = tries_histo_src.attachHdrDeltaHistogram(); - - sampler.addDirect("target_rate", cyclerate_gauge::getValue, Double.NaN); - sampler.addDeltaTime("achieved_oprate", result_timer::getCount, Double.NaN); - sampler.addDeltaTime("achieved_ok_oprate", result_success_timer::getCount, 1.0); - - sampler.addRemix("retries_p99", (vars) -> { - double triesP99 = tries_histo.getDeltaSnapshot(90).get99thPercentile(); - if (Double.isNaN(triesP99) || Double.isInfinite(triesP99) || triesP99 == 0.0d) { - return 1.0d; - } - return 1 / triesP99; - }); -// sampler.addDirect("latency_cutoff_50", () -> { -// double latencyP99 = (latency_histo.getDeltaSnapshot(90).getValue(settings.cutoff_quantile())) / 1_000_000d; -// double v = (StatFunctions.sigmoidE4LowPass(latencyP99, settings.cutoff_ms())); -// return v; -// }, 1.0d); - return sampler; +// while (frameParams != null) { +// applyParams(frameParams,flywheel); +// capture.startWindow(); +// capture.awaitSteadyState(); +// capture.stopWindow(); +// journal.record(frameParams, capture.last()); +// stdout.println(capture.last()); +// stdout.println("-".repeat(40)); +// frameParams = nextStep(journal); +// } +// return journal.bestRun().params(); + return null; + } } } diff --git a/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxSearchSettings.java b/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxConfig.java similarity index 87% rename from nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxSearchSettings.java rename to nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxConfig.java index 806739d3e..c3fe66ba7 100644 --- a/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxSearchSettings.java +++ b/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxConfig.java @@ -18,7 +18,7 @@ package io.nosqlbench.scenarios.simframe.optimizers.findmax; import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams; -public record FindmaxSearchSettings( +public record FindmaxConfig( double sample_time_ms, double sample_max, double sample_incr, @@ -26,10 +26,9 @@ public record FindmaxSearchSettings( double rate_step, double rate_incr, double average_of, - double min_settling_ms, - FindmaxParamModel model + double min_settling_ms ) { - public FindmaxSearchSettings(NBCommandParams params, FindmaxParamModel model) { + public FindmaxConfig(NBCommandParams params) { this( params.maybeGet("sample_time_ms").map(Double::parseDouble).orElse(4000d), params.maybeGet("sample_max").map(Double::parseDouble).orElse(10000d), @@ -38,8 +37,7 @@ public record FindmaxSearchSettings( params.maybeGet("rate_step").map(Double::parseDouble).orElse(100d), params.maybeGet("rate_incr").map(Double::parseDouble).orElse(2d), params.maybeGet("average_of").map(Double::parseDouble).orElse(2d), - params.maybeGet("min_settling_ms").map(Double::parseDouble).orElse(4000d), - model + params.maybeGet("min_settling_ms").map(Double::parseDouble).orElse(4000d) ); } } diff --git a/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxFrameFunction.java b/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxFrameFunction.java index 38680c39b..b8eca39b4 100644 --- a/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxFrameFunction.java +++ b/nbr/src/main/java/io/nosqlbench/scenarios/simframe/optimizers/findmax/FindmaxFrameFunction.java @@ -21,38 +21,44 @@ import io.nosqlbench.engine.api.activityapi.core.RunState; import io.nosqlbench.engine.core.lifecycle.scenario.container.ContainerActivitiesController; import io.nosqlbench.scenarios.simframe.capture.SimFrameCapture; import io.nosqlbench.scenarios.simframe.capture.SimFrameJournal; +import io.nosqlbench.scenarios.simframe.planning.HoldAndSample; import io.nosqlbench.scenarios.simframe.planning.SimFrameFunction; +import java.io.PrintWriter; + public class FindmaxFrameFunction implements SimFrameFunction { private final Activity flywheel; private final SimFrameCapture capture; private final SimFrameJournal journal; - private final FindmaxSearchSettings settings; + private final FindmaxConfig settings; private final ContainerActivitiesController controller; + private final FindmaxParamModel model; public FindmaxFrameFunction( ContainerActivitiesController controller, - FindmaxSearchSettings settings, + FindmaxConfig settings, Activity flywheel, SimFrameCapture capture, - SimFrameJournal journal + SimFrameJournal journal, + FindmaxParamModel model ) { this.controller = controller; this.settings = settings; this.flywheel = flywheel; this.capture = capture; this.journal = journal; + this.model = model; } @Override public double value(double[] point) { System.out.println("━".repeat(40)); - FindmaxFrameParams params = settings.model().apply(point); + FindmaxFrameParams params = model.apply(point); System.out.println(params); capture.startWindow(); capture.awaitSteadyState(); - settings.model().apply(point); + model.apply(point); capture.restartWindow(); System.out.println("sampling for " + settings.sample_time_ms()+"ms"); controller.waitMillis((long) settings.sample_time_ms());