IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

partial work on nb webdriver

Browse Source
This commit is contained in:
Jonathan Shook
2020-04-10 16:00:19 -05:00
parent 5e8c5bdfca
commit 115b0260f2
19 changed files with 454 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/AsyncAction.java
View File

@@ -22,10 +22,8 @@ import io.nosqlbench.engine.api.activityapi.core.ops.fluent.opfacets.TrackedOp;
import java.util.function.LongFunction;
/**
* <p>An AsyncAction allows an activity type to implement asynchronous
* An AsyncAction allows an activity type to implement asynchronous
* operations within each thread.
* </p>
*
*/
public interface AsyncAction<D> extends Action {
@@ -64,33 +62,4 @@ public interface AsyncAction<D> extends Action {
*/
boolean enqueue(TrackedOp<D> opc);
// /**
// * Await completion of all pending operations for this thread.
// * If all tasks are already complete when this is called, then it
// * should return immediately.
// * @param timeout Timeout in milliseconds
// * @return true, if all tasks pending for this thread are completed.
// */
// boolean awaitCompletion(long timeout);
// /**
// * Once constructed, all async actions are expected to provide a tracker
// * object which can be used to register callback for operational events,
// * as well as to provide a diagnostic view of what is happening with
// * the number of pending operations per thread.
// * @return An async operations tracker
// */
// OpTracker<D> getTracker();
// /**
// * When the activity needs to create a new op context which tracks all
// * things interesting for the operation, it will call this method.
// * The payload type D determines any and all of what an async action
// * may know about an op.
// *
// * @return A new op state of parameterized type D
// */
// D allocateOpData(long cycle);
}

6
engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/ratelimits/RateLimiterPerfTestMethods.java
View File

@@ -18,9 +18,9 @@
package io.nosqlbench.engine.api.activityapi.ratelimits;
import io.nosqlbench.engine.api.metrics.DeltaHdrHistogramReservoir;
import io.nosqlbench.testutils.Bounds;
import io.nosqlbench.testutils.Perf;
import io.nosqlbench.testutils.Result;
import io.nosqlbench.nb.api.testutils.Bounds;
import io.nosqlbench.nb.api.testutils.Perf;
import io.nosqlbench.nb.api.testutils.Result;
import java.util.ArrayList;
import java.util.Arrays;

4
engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/ratelimits/TestHybridRateLimiterPerf.java
View File

@@ -18,8 +18,8 @@
package io.nosqlbench.engine.api.activityapi.ratelimits;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.testutils.Perf;
import io.nosqlbench.testutils.Result;
import io.nosqlbench.nb.api.testutils.Perf;
import io.nosqlbench.nb.api.testutils.Result;
import org.junit.Ignore;
import org.junit.Test;

2
engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/ratelimits/TestRateLimiterPerf1E7.java
View File

@@ -18,7 +18,7 @@
package io.nosqlbench.engine.api.activityapi.ratelimits;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.testutils.Perf;
import io.nosqlbench.nb.api.testutils.Perf;
import org.junit.Ignore;
import org.junit.Test;

2
engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/ratelimits/TestRateLimiterPerf1E8.java
View File

@@ -18,7 +18,7 @@
package io.nosqlbench.engine.api.activityapi.ratelimits;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.testutils.Perf;
import io.nosqlbench.nb.api.testutils.Perf;
import org.junit.Ignore;
import org.junit.Test;

2
engine-api/src/test/java/io/nosqlbench/engine/api/activityapi/ratelimits/TestRateLimiterPerfSingle.java
View File

@@ -18,7 +18,7 @@
package io.nosqlbench.engine.api.activityapi.ratelimits;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.testutils.Result;
import io.nosqlbench.nb.api.testutils.Result;
import org.junit.Ignore;
import org.junit.Test;

60
engine-api/src/test/java/io/nosqlbench/testutils/Bounds.java
View File

@@ -1,60 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.testutils;
public class Bounds {
private final int levelsPerMagnitude;
private long currentValue;
public Bounds(long startingValue, int levelsPerMagnitude) {
this.currentValue=startingValue;
this.levelsPerMagnitude = levelsPerMagnitude;
}
public Bounds setValue(long value) {
this.currentValue = value;
return this;
}
public long getValue() {
return currentValue;
}
public long getNextValue() {
long nextValue = findNextHigherValue();
currentValue=nextValue;
return currentValue;
}
private long findNextHigherValue() {
int pow10 = (int) Math.log10(currentValue);
if (levelsPerMagnitude==1) {
return (long) Math.pow(10,pow10+1);
}
double baseMagnitude = Math.pow(10, pow10);
double increment = baseMagnitude/ levelsPerMagnitude;
long newValue = (long) (currentValue + increment);
return newValue;
}
@Override
public String toString() {
return String.valueOf(this.currentValue) + "(incr by 1/" + this.levelsPerMagnitude + ")";
}
}

49
engine-api/src/test/java/io/nosqlbench/testutils/BoundsTest.java
View File

@@ -1,49 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.testutils;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class BoundsTest {
@Test
public void testProgression2() {
Bounds bounds = new Bounds(3000, 2);
assertThat(bounds.getValue()).isEqualTo(3000L);
assertThat(bounds.getNextValue()).isEqualTo(3500L);
assertThat(bounds.getNextValue()).isEqualTo(4000L);
assertThat(bounds.getNextValue()).isEqualTo(4500L);
assertThat(bounds.setValue(9500).getNextValue()).isEqualTo(10000L);
assertThat(bounds.getNextValue()).isEqualTo(15000L);
}
@Test
public void testProgression1() {
Bounds bounds = new Bounds(100, 1);
assertThat(bounds.getValue()).isEqualTo(100L);
assertThat(bounds.getNextValue()).isEqualTo(1000L);
assertThat(bounds.getNextValue()).isEqualTo(10000L);
assertThat(bounds.getNextValue()).isEqualTo(100000L);
assertThat(bounds.getNextValue()).isEqualTo(1000000L);
}
}

182
engine-api/src/test/java/io/nosqlbench/testutils/Perf.java
View File

@@ -1,182 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.testutils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.DoubleStream;
/**
* Perf is a testing utility class that collects and analyzes
* performance data from individual test runs.
*/
public class Perf implements Iterable<Result> {
private final String description;
private List<Result> results = new ArrayList<>();
public Perf(String description) {
this.description = description;
}
/**
* Get the differences between successive test runs for a given
* property. The values provided have the same size as the results,
* but the first result will always be Double.NaN. This makes it
* easy to takeUpTo the results in tabular form and display them
* "as of" a given result index.
*
* @param resultProperty A function that yields a double from a Result
* @return an array of deltas of that property
*/
public double[] getDeltas(Function<Result, Double> resultProperty) {
double[] values = new double[results.size()];
for (int i = 0; i < results.size(); i++) {
values[i] = (i == 0) ? Double.NaN : resultProperty.apply(results.get(i)) - resultProperty.apply(results.get(i - 1));
}
return values;
}
/**
* Add a test result to this performance collector.
* @param result a {@link Result} object
* @return this Perf, for method chaining
*/
public Perf add(Result result) {
this.results.add(result);
return this;
}
/**
* Add a test result to this performance collector.
* @param description A description of the result
* @param start The start time of the test run
* @param end The end time of the test run
* @param ops The total number of iterations of the test run
* @return this Perf, for method chaining
*/
public Perf add(String description, long start, long end, long ops) {
return this.add(new Result(description, start, end, ops));
}
/**
* Extract the double field value from the last results and return whether or not
* they are within some fractional margin between the minimum and maximum seen value.
* @param resultProperty A function to extract the double field value
* @param withinMargin A value like 0.01f to represent "10 percent"
* @param count The number of recent results that must be present
* @return true if there are at least count results, and the min and max values are within that margin
*/
public boolean isConverged(Function<Result, Double> resultProperty, double withinMargin, int count) {
if (results.size() < (count + 1)) {
return false;
}
double actualMargin = getMaximumDelta(resultProperty, count);
return (actualMargin < withinMargin);
}
/**
* For the most recent test runs, measure the maximum difference in
* a given property.
* @param resultProperty A function that produces a property from a {@link Result}
* @param count The number of recent test runs to consider
* @return The difference between the min and max values of the property
*/
public double getMaximumDelta(Function<Result, Double> resultProperty, int count) {
if (results.size() < (count + 1)) {
return Double.NaN;
}
double[] values = results.stream().skip(results.size()-count).map(resultProperty).mapToDouble(Double::doubleValue).toArray();
double min = DoubleStream.of(values).min().orElse(Double.MAX_VALUE);
double max = DoubleStream.of(values).max().orElse(Double.MIN_VALUE);
return (max-min) / max;
}
/**
* Sort the internal results according to some property
* @param resultProperty A function that produces a property from a {@link Result}
* @return this Perf, for method chaining
*/
public Perf sort(Function<Result, Double> resultProperty) {
results = results.stream().sorted(Comparator.comparing(resultProperty)).collect(Collectors.toList());
return this;
}
public String toString() {
StringBuilder sb = new StringBuilder(this.description + "\n");
results.forEach(r -> {
sb.append(r);
sb.append("\n");
});
return sb.toString();
}
/**
* Summarize the last results in a tabular format, with row-by-row delta included
* for a given property.
* @param resultProperty A function that extracts a property from a {@link Result}
* @param deltaDescription The description of the delta column
* @param lastN The number of recent test runs to include
* @return A tabular representation of the test runs and the deltas for the property
*/
public String toStringDelta(Function<Result, Double> resultProperty, String deltaDescription, int... lastN) {
int count = (lastN.length==1 ? lastN[0] : results.size());
double[] deltas = getDeltas(resultProperty);
List<String> pvalues = DoubleStream.of(deltas).mapToObj(v -> String.format("%-10.3f", v)).collect(Collectors.toList());
List<String> rvalues = Result.toString(results);
int maxlen = pvalues.stream().mapToInt(String::length).max().orElse(0);
maxlen = Math.max(maxlen,deltaDescription.length());
StringBuilder sb = new StringBuilder(String.format("iter %-" + maxlen + "s %s\n", deltaDescription, this.description));
String rowfmt = "%03d: %-" + maxlen + "s %s\n";
for (int i = 0; i < results.size(); i++) {
sb.append(String.format(rowfmt, i, pvalues.get(i), rvalues.get(i)));
}
return sb.toString();
}
/**
* @return Returns the last result
*/
public Result getLastResult() {
return results.get(results.size() - 1);
}
/**
* Reduce a number of independent and concurrent runs into a single summary.
* @return A Perf with a single result
*/
public Perf reduceConcurrent() {
long totalOps = results.stream().mapToLong(Result::getTotalOps).sum();
double avgStart = results.stream().mapToLong(Result::getStartNanos).average().orElse(Double.NaN);
double avgEnd = results.stream().mapToLong(Result::getEndNanos).average().orElse(Double.NaN);
return new Perf("summary of \" + results.size() + \" concurrent results\"")
.add("summary of " + results.size() + " concurrent results", (long)avgStart, (long)avgEnd, totalOps);
}
@Override
public Iterator<Result> iterator() {
return results.iterator();
}
}

43
engine-api/src/test/java/io/nosqlbench/testutils/PerfTest.java
View File

@@ -1,43 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.testutils;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class PerfTest {
@Test
public void testBasics() {
Perf p = new Perf("unittest");
p.add("0",0,100,100);
assertThat(p.isConverged(Result::getOpsPerSec,0.2d, 3)).isFalse();
p.add("1",0,100,121);
assertThat(p.isConverged(Result::getOpsPerSec,0.2d, 3)).isFalse();
p.add("2",0,100,1421);
assertThat(p.isConverged(Result::getOpsPerSec,0.2d, 3)).isFalse();
p.add("3",0,100,1431);
double[] deltas = p.getDeltas(Result::getOpsPerSec);
System.out.println("Sanity Check for Perf methods:\n"+p.toStringDelta(Result::getOpsPerSec, "D_ops_s"));
assertThat(p.isConverged(Result::getOpsPerSec,0.2d, 3)).isFalse();
p.add("4",0,100,1441);
assertThat(p.isConverged(Result::getOpsPerSec,0.2d, 3)).isTrue();
}
}

94
engine-api/src/test/java/io/nosqlbench/testutils/Result.java
View File

@@ -1,94 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.testutils;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class Result {
private final long start;
private final long end;
private final long ops;
private String description;
public Result(String description, long start, long end, long ops) {
this.description = description;
this.start = start;
this.end = end;
this.ops = ops;
}
public String getDescription() {
return description;
}
public long getTotalOps() {
return ops;
}
public double getTimeSeconds() {
return (double) (end - start) / 1_000_000_000d;
}
public double getNsPerOp() {
return (double) (end - start) / (double) ops;
}
public double getOpsPerSec() {
return (double) getTotalOps() / getTimeSeconds();
}
@Override
public String toString() {
long time_ns = end - start;
return String.format("'%s': %d_ops %f_S %.3f_ops_s, %.0f_ns_op", description, ops, getTimeSeconds(), getOpsPerSec(), getNsPerOp());
}
public static List<String> toString(List<Result> results) {
List<String> ldesc = results.stream().map(Result::getDescription).collect(Collectors.toList());
List<String> lops = results.stream().map(r -> String.format("%d_ops",r.getTotalOps())).collect(Collectors.toList());
List<String> ltime_s = results.stream().map(r -> String.format("%f_S",r.getTimeSeconds())).collect(Collectors.toList());
List<String> lops_s = results.stream().map(r -> String.format("%.3f_ops_s",r.getOpsPerSec())).collect(Collectors.toList());
List<String> lns_op = results.stream().map(r -> String.format("%.0f_ns_op",r.getNsPerOp())).collect(Collectors.toList());
int sizeof_ldesc = ldesc.stream().mapToInt(String::length).max().orElse(0);
int sizeof_lops = lops.stream().mapToInt(String::length).max().orElse(0);
int sizeof_ltime_s = ltime_s.stream().mapToInt(String::length).max().orElse(0);
int sizeof_lops_s = lops_s.stream().mapToInt(String::length).max().orElse(0);
int sizeof_lns_op = lns_op.stream().mapToInt(String::length).max().orElse(0);
String fmt = "'%" + sizeof_ldesc + "s': %" + sizeof_lops + "s %" + sizeof_ltime_s + "s %" + sizeof_lops_s + "s %" + sizeof_lns_op + "s";
List<String> rows = new ArrayList<>(results.size());
for (int i = 0; i < ldesc.size(); i++) {
String row = String.format(fmt, ldesc.get(i), lops.get(i), ltime_s.get(i), lops_s.get(i), lns_op.get(i));
rows.add(row);
}
return rows;
}
public long getStartNanos() {
return this.start;
}
public long getEndNanos() {
return this.end;
}
}