incremental progress

nbr/src/main/java/io/nosqlbench/scenarios/BasePerfDimension.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import java.util.function.DoubleSupplier;
+
+public abstract class BasePerfDimension implements PerfDimension {
+    private double weight;
+    private DoubleSupplier supplier;
+    private String name;
+    private Weighting weighting = Weighting.uniform;
+
+    public BasePerfDimension(String name, double weight, Weighting weighting, DoubleSupplier supplier) {
+        this.weight = weight;
+        this.supplier = supplier;
+        this.name = name;
+        this.weighting = weighting;
+    }
+
+    @Override
+    public double getWeight() {
+        return this.weight;
+    }
+
+    @Override
+    public DoubleSupplier getSupplier() {
+        return supplier;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public Weighting getWeighting() {
+        return weighting;
+    }
+
+    @Override
+    public abstract String getValue();
+}

nbr/src/main/java/io/nosqlbench/scenarios/PerfDimension.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import java.util.function.DoubleSupplier;
+
+public interface PerfDimension {
+
+    public double getWeight();
+
+    public Weighting getWeighting();
+    public DoubleSupplier getSupplier();
+    public String getName();
+    public String getValue();
+
+}

nbr/src/main/java/io/nosqlbench/scenarios/PerfWindowSampler.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.DoubleSupplier;
+
+/**
+ * This is a helper class that makes it easy to bundle up a combination of measurable
+ * factors and get a windowed sample from them. To use it, add your named data sources
+ * with their coefficients, and optionally a callback which resets the measurement
+ * buffers for the next time. When you call {@link #getCurrentWindowValue()}, all callbacks
+ * are used after the value computation is complete.
+ *
+ * <P>This is NOT thread safe!</P>
+ */
+public class PerfWindowSampler {
+
+    private final List<Criterion> criteria = new ArrayList<>();
+    private boolean openWindow = false;
+
+    private final static int STARTS = 0;
+    private final static int ENDS = 1;
+    private final static int WEIGHTED = 2;
+    private final static int START_TIME = 3;
+    private final static int END_TIME = 4;
+    private final static int ARYSIZE = END_TIME+1;
+    /**
+     * window, measure, START,STOP,WEIGHTED
+     */
+    private double[][][] data;
+    private int window = -1;
+
+
+    void addDirect(String name, DoubleSupplier supplier, double weight, Runnable callback) {
+        this.criteria.add(new Criterion(name, supplier, weight, callback, false));
+    }
+
+    void addDirect(String name, DoubleSupplier supplier, double weight) {
+        addDirect(name, supplier, weight, () -> {
+        });
+    }
+
+    void addDeltaTime(String name, DoubleSupplier supplier, double weight, Runnable callback) {
+        this.criteria.add(new Criterion(name, supplier, weight, callback, true));
+    }
+
+    void addDeltaTime(String name, DoubleSupplier supplier, double weight) {
+        addDeltaTime(name, supplier, weight, () -> {
+        });
+    }
+
+    double getCurrentWindowValue() {
+        if (openWindow) {
+            throw new RuntimeException("invalid access to checkpoint value on open window.");
+        }
+        double product = 1.0d;
+        if (data==null) {
+            return Double.NaN;
+        }
+        double[][] values = data[window];
+
+        for (int i = 0; i < criteria.size(); i++) {
+            product *= values[i][WEIGHTED];
+        }
+        return product;
+    }
+    private double valueOf(int measuredItem) {
+        double[] vals = data[window][measuredItem];
+
+        if (criteria.get(measuredItem).delta) {
+            return (vals[ENDS] - vals[STARTS]) / (vals[END_TIME] - vals[START_TIME])*1000.0d;
+        } else {
+            return vals[ENDS];
+        }
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder("PERF " + (openWindow ? "OPENWINDOW! " : "" ) + "sampler value =").append(getCurrentWindowValue()).append("\n");
+        for (int i = 0; i < criteria.size(); i++) {
+            Criterion criterion = criteria.get(i);
+            sb.append("->").append(criterion.name).append(" last=").append(valueOf(i)).append("\n");
+        }
+        return sb.toString();
+    }
+
+    public void startWindow() {
+        startWindow(System.currentTimeMillis());
+
+    }
+    public void startWindow(long now) {
+        openWindow=true;
+        window++;
+        if (this.data == null) {
+            this.data = new double[1][criteria.size()][ARYSIZE];
+        }
+        if (this.window >=data.length) {
+            double[][][] newary = new double[data.length<<1][criteria.size()][ARYSIZE];
+            System.arraycopy(data,0,newary,0,data.length);
+            this.data = newary;
+        }
+        for (int i = 0; i < criteria.size(); i++) {
+            data[window][i][START_TIME] = now;
+            Criterion criterion = criteria.get(i);
+            if (criterion.delta) {
+                data[window][i][STARTS] = criterion.supplier.getAsDouble();
+            } else {
+                data[window][i][STARTS] = Double.NaN;
+            }
+            criterion.callback.run();
+        }
+        for (Criterion criterion : criteria) {
+            criterion.callback.run();
+        }
+    }
+
+    public void stopWindow() {
+        stopWindow(System.currentTimeMillis());
+    }
+    public void stopWindow(long now) {
+        for (int i = 0; i < criteria.size(); i++) {
+            data[window][i][END_TIME] = now;
+            Criterion criterion = criteria.get(i);
+            double endmark = criterion.supplier.getAsDouble();
+            data[window][i][ENDS] = endmark;
+
+            double sample = valueOf(i);
+            data[window][i][WEIGHTED] = sample* criterion.weight;
+        }
+        openWindow=false;
+    }
+
+    public static record Criterion(
+        String name,
+        DoubleSupplier supplier,
+        double weight,
+        Runnable callback,
+        boolean delta
+    ) { }
+}

nbr/src/main/java/io/nosqlbench/scenarios/SC_optimo.java

@@ -18,7 +18,9 @@ package io.nosqlbench.scenarios;
  */
 
 
-import io.nosqlbench.api.engine.metrics.instruments.NBMetric;
+import io.nosqlbench.api.engine.metrics.ConvenientSnapshot;
+import io.nosqlbench.api.engine.metrics.DeltaSnapshotReader;
+import io.nosqlbench.api.engine.metrics.instruments.NBMetricTimer;
 import io.nosqlbench.api.optimizers.BobyqaOptimizerInstance;
 import io.nosqlbench.api.optimizers.MVResult;
 import io.nosqlbench.components.NBComponent;
@@ -27,12 +29,15 @@ import io.nosqlbench.engine.core.lifecycle.scenario.direct.SCBaseScenario;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.ToDoubleFunction;
 
 public class SC_optimo extends SCBaseScenario {
     private final static Logger logger = LogManager.getLogger(SC_optimo.class);
+
     public SC_optimo(NBComponent parentComponent, String scenarioName) {
         super(parentComponent, scenarioName);
     }
@@ -42,53 +47,82 @@ public class SC_optimo extends SCBaseScenario {
         // TODO: having "scenario" here as well as in "named scenario" in workload templates is confusing. Make this clearer.
         String workload = params.getOrDefault("workload", "default_workload");
 
-        Map<String,String> activityParams = new HashMap<>(Map.of(
+        Map<String, String> activityParams = new HashMap<>(Map.of(
             "cycles", String.valueOf(Long.MAX_VALUE),
             "threads", "1",
             "driver", "diag",
             "rate", "1"
         ));
         if (params.containsKey("workload")) {
-            activityParams.put("workload",params.get("workload"));
+            activityParams.put("workload", params.get("workload"));
         } else if (params.containsKey("op")) {
-            activityParams.put("op",params.get("op"));
+            activityParams.put("op", params.get("op"));
         } else {
-            activityParams.put("op","log: level=info");
+            activityParams.put("op", "log: level=info");
             logger.warn("You provided neither a workload nor an op, so assuming diagnostic mode.");
         }
 
-        Activity flywheel = controller.start(activityParams);
+        int seconds = params.containsKey("window") ? Integer.parseInt(params.get("window")) : 5;
 
         BobyqaOptimizerInstance bobby = create().bobyqaOptimizer();
-
-        bobby.param("threads", 0.0d, 200000.0d);
-        bobby.param("rate", 0.0d, 1_000_000.d);
+        bobby.param("rate", 1.0d, 10000.d);
+        bobby.param("threads", 1.0d, 1000.0d);
         bobby.setInitialRadius(10000.0).setStoppingRadius(0.001).setMaxEval(1000);
 
+        Activity flywheel = controller.start(activityParams);
+        stdout.println("warming up for " + seconds + " seconds");
+        controller.waitMillis(5000);
+
+
         /**
          * <P>This function is the objective function, and is responsible for applying
          * the parameters and yielding a result. The higher the returned result, the
          * better the parameters are.</P>
          * <P>The parameter values will be passed in as an array, pair-wise with the param calls above.</P>
          */
+
+        PerfWindowSampler sampler = new PerfWindowSampler();
+        NBMetricTimer result_success_timer = flywheel.find().timer("name:ressult_success");
+        sampler.addDeltaTime("achieved_rate", result_success_timer::getCount, 1.0);
+        final DeltaSnapshotReader snapshotter = result_success_timer.getDeltaReader();
+        AtomicReference<ConvenientSnapshot> snapshot = new AtomicReference<>(snapshotter.getDeltaSnapshot());
+        ValidAtOrBelow below15000 = ValidAtOrBelow.max(15000);
+        sampler.addDirect(
+            "p99latency",
+            () -> below15000.applyAsDouble(snapshot.get().getP99ns()),
+            -1.0,
+            () -> snapshot.set(snapshotter.getDeltaSnapshot())
+        );
+        sampler.startWindow();
+
         ToDoubleFunction<double[]> f = new ToDoubleFunction<double[]>() {
             @Override
-            public double applyAsDouble(double[] value) {
-                int threads=(int)value[0];
-
-                NBMetric counter = flywheel.find().counter("counterstuff");
+            public double applyAsDouble(double[] values) {
+                stdout.println("params=" + Arrays.toString(values));
+                int threads = (int) bobby.getParams().getValue("threads", values);
+                double rate = bobby.getParams().getValue("rate", values);
 
+                stdout.println("setting threads to " + threads);
                 flywheel.getActivityDef().setThreads(threads);
 
-                double rate=value[1];
-                flywheel.getActivityDef().setCycles(String.valueOf(rate));
+                String ratespec = rate + ":1.1:restart";
+                stdout.println("setting rate to " + ratespec);
+                flywheel.getActivityDef().getParams().put("rate", ratespec);
 
+                sampler.startWindow();
+                stdout.println("waiting " + seconds + " seconds...");
+                controller.waitMillis(seconds * 1000L);
+                sampler.stopWindow();
+                double value = sampler.getCurrentWindowValue();
+                stdout.println(sampler.toString());
+                return value;
 
-                return 10000000 - ((Math.abs(100-value[0])) + (Math.abs(100-value[1])));
             }
         };
         bobby.setObjectiveFunction(f);
         MVResult result = bobby.optimize();
+
+        controller.stop(flywheel);
         stdout.println("optimized result was " + result);
         stdout.println("map of result was " + result.getMap());
 

nbr/src/main/java/io/nosqlbench/scenarios/Uniform.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import java.util.function.DoubleSupplier;
+
+public class Uniform extends BasePerfDimension {
+
+    public Uniform(double weight, DoubleSupplier supplier, String name) {
+        super(name, weight, Weighting.uniform, supplier);
+    }
+
+    @Override
+    public String getValue() {
+        return null;
+    }
+}

nbr/src/main/java/io/nosqlbench/scenarios/ValidAtOrAbove.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import jakarta.validation.Valid;
+
+import java.util.function.DoubleUnaryOperator;
+
+public class ValidAtOrAbove implements DoubleUnaryOperator {
+
+    public ValidAtOrAbove(double threshold, double defaultValue) {
+        this.threshold = threshold;
+        this.defaultValue = defaultValue;
+    }
+
+    private double threshold;
+    private double defaultValue;
+
+    @Override
+    public double applyAsDouble(double operand) {
+        if (operand>=threshold) {
+            return operand;
+        } else {
+            return defaultValue;
+        }
+    }
+
+    public static ValidAtOrAbove min(double min) {
+        return new ValidAtOrAbove(min,0.0d);
+    }
+}

nbr/src/main/java/io/nosqlbench/scenarios/ValidAtOrBelow.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import java.util.function.DoubleUnaryOperator;
+
+public class ValidAtOrBelow implements DoubleUnaryOperator {
+
+    public ValidAtOrBelow(double threshold, double defaultValue) {
+        this.threshold = threshold;
+        this.defaultValue = defaultValue;
+    }
+
+    private double threshold;
+    private double defaultValue;
+
+    @Override
+    public double applyAsDouble(double operand) {
+        if (operand<=threshold) {
+            return operand;
+        } else {
+            return defaultValue;
+        }
+    }
+
+    public static ValidAtOrBelow max(double max) {
+        return new ValidAtOrBelow(max,0.0d);
+    }
+}

nbr/src/main/java/io/nosqlbench/scenarios/ValueFuncs.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+public class ValueFuncs {
+
+    public static double zeroBelow(double value, double threshold) {
+        if (value<threshold) {
+            return 0.0d;
+        }
+        return value;
+    }
+
+    public static double zeroAbove(double value, double threshold) {
+        if (value>threshold) {
+            return 0.0d;
+        }
+        return value;
+    }
+
+    /**
+     * Apply exponential weighting to the value base 2. For rate=1.0, the weight
+     */
+    public static double exp_2(double value) {
+        return (value*value)+1;
+    }
+
+    public static double exp_e(double value) {
+        return Math.exp(value);
+    }
+}

nbr/src/main/java/io/nosqlbench/scenarios/Weighting.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+public enum Weighting {
+    uniform;
+    public double applyWeight(double input) {
+        return switch (this) {
+            case uniform -> input;
+        };
+    }
+}

nbr/src/main/java/io/nosqlbench/scenarios/WindowSampler.java

@@ -1,36 +0,0 @@
-package io.nosqlbench.scenarios;
-
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import io.nosqlbench.components.NBComponent;
-
-public class WindowSampler {
-    private final NBComponent base;
-
-    public WindowSampler(NBComponent component) {
-        this.base = component;
-        component.find().metric("doesnot=exist");
-    }
-
-    public Sample sample() {
-        return new Sample(1.0d,2.0d);
-    }
-
-    public static record Sample(double rate, double p99) { }
-}

nbr/src/test/java/io/nosqlbench/scenarios/PerfWindowSamplerTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.scenarios;
+
+import org.assertj.core.data.Offset;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.DoubleSupplier;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class PerfWindowSamplerTest {
+
+    @Test
+    public void testBasicValues() {
+        PerfWindowSampler pws = new PerfWindowSampler();
+        pws.addDirect("a",() -> 1.0d, 1.0d);
+        pws.addDirect("b",()-> 3.0d, 3.0d);
+
+        pws.startWindow();
+        pws.stopWindow();
+        double value = pws.getCurrentWindowValue();
+        assertThat(value).isCloseTo(9.0, Offset.offset(0.002));
+    }
+
+    @Test
+    public void testDeltaValues() {
+        AtomicLong a1 = new AtomicLong(0);
+        DoubleSupplier ds1 = () -> (double) a1.get();
+
+        AtomicLong a2 = new AtomicLong(0);
+        DoubleSupplier ds2 = () -> (double) a2.get();
+
+        PerfWindowSampler pws = new PerfWindowSampler();
+        pws.addDeltaTime("a",ds1, 1.0d);
+        pws.addDeltaTime("b",ds2, 1.0d);
+
+        pws.startWindow(0L);
+
+        a1.set(3L);
+        a2.set(10L);
+
+        pws.stopWindow(1000L);
+        double value = pws.getCurrentWindowValue();
+        assertThat(value).isCloseTo(30.0,Offset.offset(0.001));
+
+        pws.startWindow(10000L);
+        a1.set(42); // 42-10=32
+        a2.set(42); // 42-1=41; 41+32=73
+
+        pws.stopWindow(11000L);
+        double value2 = pws.getCurrentWindowValue();
+        assertThat(value2).isCloseTo(1248.0,Offset.offset(0.001));
+
+    }
+
+}