diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml
index e7685b19d..4299d76d7 100644
--- a/mvn-defaults/pom.xml
+++ b/mvn-defaults/pom.xml
@@ -639,6 +639,7 @@
+ --enable-preview @{argLine}
1
false
@@ -903,6 +904,7 @@
org.apache.maven.plugins
maven-javadoc-plugin
+ --enable-preview
21
${javadoc.name}
${javadoc.name}
@@ -911,6 +913,7 @@
false
+
-Xdoclint:none
diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java
index b443117fa..6da85376e 100644
--- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java
+++ b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java
@@ -16,16 +16,16 @@
package io.nosqlbench.adapter.http;
-import io.nosqlbench.adapter.http.core.HttpFormatParser;
-import io.nosqlbench.adapter.http.core.HttpOp;
-import io.nosqlbench.adapter.http.core.HttpOpMapper;
-import io.nosqlbench.adapter.http.core.HttpSpace;
+import io.nosqlbench.adapter.http.core.*;
+import io.nosqlbench.nb.api.components.core.NBComponentProps;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.Param;
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
+import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
+import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@@ -41,8 +41,16 @@ import java.util.function.Function;
@Service(value = DriverAdapter.class, selector = "http")
public class HttpDriverAdapter extends BaseDriverAdapter {
+ public final NBMetricHistogram statusCodeHistogram;
+
public HttpDriverAdapter(NBComponent parent, NBLabels labels) {
super(parent, labels);
+ this.statusCodeHistogram = create().histogram(
+ "statuscode",
+ Integer.parseInt(getComponentProp(NBComponentProps.HDRDIGITS).orElse("3")),
+ MetricCategory.Payload,
+ "A histogram of status codes received by the HTTP client"
+ );
}
@Override
@@ -54,7 +62,7 @@ public class HttpDriverAdapter extends BaseDriverAdapter {
@Override
public Function getSpaceInitializer(NBConfiguration cfg) {
- return spaceName -> new HttpSpace(getParent(), spaceName, cfg);
+ return spaceName -> new HttpSpace(this, spaceName, cfg);
}
@Override
@@ -83,5 +91,4 @@ public class HttpDriverAdapter extends BaseDriverAdapter {
return super.getConfigModel().add(HttpSpace.getConfigModel()).add(thisCfgModel);
}
-
}
diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpMetrics.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpMetrics.java
deleted file mode 100644
index 8cd4a301a..000000000
--- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpMetrics.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2022-2023 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.adapter.http.core;
-
-import com.codahale.metrics.Histogram;
-import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
-import io.nosqlbench.nb.api.labels.NBLabeledElement;
-import io.nosqlbench.nb.api.labels.NBLabels;
-import io.nosqlbench.nb.api.components.core.NBComponent;
-
-public class HttpMetrics implements NBLabeledElement {
- private final NBComponent parent;
- private final HttpSpace space;
- final Histogram statusCodeHistogram;
-
- public HttpMetrics(NBComponent parent, HttpSpace space) {
- this.parent = parent;
- this.space = space;
- statusCodeHistogram = parent.create().histogram(
- "statuscode",
- space.getHdrDigits(),
- MetricCategory.Payload,
- "A histogram of status codes received by the HTTP client"
- );
- }
-
- public String getName() {
- return "http"+("default".equals(this.space.getSpaceName())?"": '-' + space.getSpaceName());
- }
-
- @Override
- public NBLabels getLabels() {
- return space.getLabels();
- }
-}
diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOp.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOp.java
index de1726237..9eb871ab4 100644
--- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOp.java
+++ b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOp.java
@@ -19,24 +19,13 @@ package io.nosqlbench.adapter.http.core;
import io.nosqlbench.adapter.http.errors.InvalidResponseBodyException;
import io.nosqlbench.adapter.http.errors.InvalidStatusCodeException;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
-import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp;
-import org.apache.logging.log4j.core.tools.picocli.CommandLine;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
-
-
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Spliterators;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
-import java.util.stream.StreamSupport;
public class HttpOp implements CycleOp {
@@ -75,7 +64,7 @@ public class HttpOp implements CycleOp {
try {
CompletableFuture> responseFuture = client.sendAsync(request, bodyreader);
response = responseFuture.get(space.getTimeoutMillis(), TimeUnit.MILLISECONDS);
- space.getHttpMetrics().statusCodeHistogram.update(response.statusCode());
+ space.statusCodeHistogram.update(response.statusCode());
if (ok_status != null) {
if (!ok_status.matcher(String.valueOf(response.statusCode())).matches()) {
diff --git a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java
index 63cffd03f..939986bc0 100644
--- a/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java
+++ b/nb-adapters/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java
@@ -16,6 +16,9 @@
package io.nosqlbench.adapter.http.core;
+import io.nosqlbench.adapter.http.HttpDriverAdapter;
+import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
+import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
@@ -39,24 +42,25 @@ import java.util.Locale;
public class HttpSpace implements NBLabeledElement {
private final static Logger logger = LogManager.getLogger(HttpSpace.class);
- private final NBComponent parent;
+ private final HttpDriverAdapter parentAdapter;
private final String name;
private final NBConfiguration cfg;
+ public NBMetricHistogram statusCodeHistogram;
private HttpConsoleFormats console;
private HttpClient.Redirect followRedirects;
private Duration timeout;
private long timeoutMillis;
private final HttpClient httpclient;
private int hdrDigits;
- private HttpMetrics httpMetrics;
private boolean diagnosticsEnabled;
- public HttpSpace(NBComponent parent, String spaceName, NBConfiguration cfg) {
- this.parent = parent;
+ public HttpSpace(HttpDriverAdapter parentAdapter, String spaceName, NBConfiguration cfg) {
+ this.parentAdapter = parentAdapter;
this.name = spaceName;
this.cfg = cfg;
applyConfig(cfg);
+ this.statusCodeHistogram = parentAdapter.statusCodeHistogram;
this.httpclient = newClient();
}
@@ -79,10 +83,9 @@ public class HttpSpace implements NBLabeledElement {
);
this.timeout = Duration.ofMillis(cfg.get("timeout", long.class));
this.timeoutMillis = cfg.get("timeout", long.class);
- this.httpMetrics = new HttpMetrics(parent, this);
this.console = cfg.getOptional("diag").map(s -> HttpConsoleFormats.apply(s, this.console))
- .orElseGet(() -> HttpConsoleFormats.apply(null,null));
+ .orElseGet(() -> HttpConsoleFormats.apply(null, null));
this.diagnosticsEnabled = console.isDiagnosticMode();
@@ -105,10 +108,6 @@ public class HttpSpace implements NBLabeledElement {
return name;
}
- public HttpMetrics getHttpMetrics() {
- return httpMetrics;
- }
-
public boolean isDiagnosticMode() {
return diagnosticsEnabled;
}
@@ -124,18 +123,15 @@ public class HttpSpace implements NBLabeledElement {
.setDescription("Whether to follow redirects. Normal redirects are those which do not " +
"redirect from HTTPS to HTTP.")
)
- .add(Param.optional(List.of("diag","diagnostics"), String.class)
+ .add(Param.optional(List.of("diag", "diagnostics"), String.class)
.setDescription("Print extended diagnostics. This option has numerous" +
" possible values. See the markdown docs for details. (nb help http)")
)
- .add(Param.defaultTo("timeout", 1000L*60L*15L) // 15 minutes
+ .add(Param.defaultTo("timeout", 1000L * 60L * 15L) // 15 minutes
.setDescription("How long to wait for requests before timeout out. Default is forever."))
.add(Param.defaultTo("hdr_digits", 4)
.setDescription("number of digits of precision to keep in HDR histograms"))
.asReadOnly();
-
}
-
-
}
diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
index 04de0e633..1d7f43595 100644
--- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
+++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
@@ -70,7 +70,7 @@ import java.util.function.LongFunction;
* specification tests under the workload_definition folder of the adapters-api module.
*
*
- *
+ *
* Op Template Parsing
*
* - Rule #1: All op templates are parsed into an internal normalized structure which contains:
@@ -92,36 +92,41 @@ import java.util.function.LongFunction;
* The net effect of these rules is that the NoSQLBench driver developer may safely use functional forms to access data
* in the op template, or may decide that certain op fields must only be provided in a static way per operation.
*
- *
+ *
* Distinguishing Op Payload from Op Config
* When a user specifies an op template, they may choose to provide only a single set of op fields without
* distinguishing between config or payload, or they may choose to directly configure each.
- *
Example:
+ *
* {@code
* ops:
- * # both op and params explicitly named
- * op1:
- * op:
- * opfield1: value1
- * params:
- * param2: value2
- * # neither op field nor params named, so all assumed to be op fields
- * op2:
- * opfield1: value1
- * param2: value2
- * # in this case, if param2 is meant to be config level,
- * # it is a likely config error that should be thrown to the user
- * # only op field explicitly named, so remainder automatically pushed into params
- * op3:
- * op:
- * opfield1: value1
- * param2: value2
- * # only params explicitly named, so remainder pushed into op payload
- * op4:
- * params:
- * param2: value2
- * opfield1: value1
- * }
+ *
+ * # both op and params explicitly named
+ * op1:
+ * op:
+ * opfield1: value1
+ * params:
+ * param2: value2
+ *
+ * # neither op field nor params named, so all assumed to be op fields
+ * op2:
+ * opfield1: value1
+ * param2: value2
+ *
+ * # in this case, if param2 is meant to be config level,
+ * # it is a likely config error that should be thrown to the user
+ * # only op field explicitly named, so remainder automatically pushed into params
+ * op3:
+ * op:
+ * opfield1: value1
+ * param2: value2
+ *
+ * # only params explicitly named, so remainder pushed into op payload
+ * op4:
+ * params:
+ * param2: value2
+ * opfield1: value1
+ * }
+ *
*
* All of these are considered valid constructions, and all of them may actually achieve the same result.
* This looks like an undesirable problem, but it serves to simplify things for users in one specific way: It allows
@@ -135,37 +140,37 @@ import java.util.function.LongFunction;
*
* Design Invariants
*
- * The above rules imply invariants, which are made explicit here. {@link ParsedOp}.
+ * The above rules imply invariants, which are made explicit here.
*
- *
+ * Single Purpose Fields
+ * You may not use an op field name or parameter name for more than one purpose.
*
- * - You may not use an op field name or parameter name for more than one purpose.
- *
- * - Treat all parameters supported by a driver adapter and it's op fields as a globally shared namespace, even if it
- * is not.
- * This avoids creating any confusion about what a parameter can be used for and how to use it for the right thing in
- * the right place.
- * For example, you may not use the parameter name `socket` in an op template to mean one thing and then use it
+ *
Shared Namespace
+ * Treat all parameters supported by a driver adapter and it's op fields as a globally shared namespace, even
+ * if it is not.
+ *
+ * This avoids creating any confusion about what a parameter can be used for and how to use it for the right thing in
+ * the right place. For example, you may not use the parameter name `socket` in an op template to mean one thing and then use it
* at the driver adapter level to mean something different. However, if the meaning is congruent, a driver developer
* may choose to support some cross-cutting parameters at the activity level. These allowances are explicit,
- * however, as each driver dictates what it will allow as activity parameters.
- *
- *
- *
+ * however, as each driver dictates what it will allow as activity parameters.
+ *
+ * Layered Resolution
+ *
+ * Users may specify op payload fields within op params or activity params as fallback config sources in that
+ * order.
*
- * - Users may specify op payload fields within op params or activity params as fallback config sources in that
- * order.
*
* - IF a name is valid as an op field, it must also be valid as such when specified in op params.
* - If a name is valid as an op field, it must also be valid as such when specified in activity params, within the
* scope of {@link ParsedOp}
* - When an op field is found via op params or activity params, it may NOT be dynamic. If dynamic values are intended
- * to be provided
- * at a common layer in the workload, then bindings support this already.
+ * to be provided at a common layer in the workload, then bindings or template variables support this already.
*
*
*
- * - You must access non-payload params via Config-oriented methods.
+ *
Configuration Fields
+ * You must access non-payload params via Config-oriented methods.
*
* - Op Templates contain op payload data and op configs (params, activity params).
* - You must use only {@link ParsedOp} getters with "...Config..." names, such as
@@ -176,56 +181,49 @@ import java.util.function.LongFunction;
*
*
*
- * - The user must be warned when a required or optional config value is missing from op params (or activity
- * params), but a value
- * of the same name is found in op payload fields.
- *
- * - If rule #1 is followed, and names are unambiguous across the driver, then it is almost certainly a configuration
- * error.
- *
- *
+ * Sanity Checks
*
- * - When both an op payload field and a param field of the same name are defined through cascading configuration
- * of param fields,
- * the local op payload field takes precedence.
+ *
The user must be warned when a required or optional config value is missing from op params (or activity
+ * params), but a value of the same name is found in op payload fields.
+ * If rule #1 is followed, and names are unambiguous across the driver, then it is almost certainly a configuration
+ * error.
+ *
+ * Precedence
+ * The order of precedence for values is:
*
- * - This is an extension of the param override rules which say that the closest (most local) value to an operation is
- * the one that takes precedence.
- * - In practice, there will be no conflicts between direct static and dynamic fields, but there will be possibly
- * between
- * static or dynamic fields and parameters and activity params. If a user wants to promote an activity param as an
- * override to existing op fields,
- * template variables allow for this to happen gracefully. Otherwise, the order of precedence is 1) op fields 2) op
- * params 3) activity params.
- *
- *
+ * - op payload fields
+ * - op param fields
+ * - activity params (when enabled, see below)
*
*
*
+ * Enabling Activity Params
+ * If a user wants to allow an activity param as an default for an fields, they must publish the op field
+ * name in the configuration model for the activity. Otherwise it is an error to specify the value at the activity
+ * level.
+ *
*
*
* Op Payload Forms
* Field values can come from multiple sources. These forms and any of their combinations are supported.
*
* Static Op Fields
- * Example:
* {@code
* op:
- * field1: value1
- * field2:
- * map3:
- * key4: value4
- * map5:
- * key6: value6
- * field7: false
- * field8: 8.8
+ * field1: value1
+ * field2:
+ * map3:
+ * key4: value4
+ * map5:
+ * key6: value6
+ * field7: false
+ * field8: 8.8
* }
*
* As shown, any literal value of any valid YAML type, including structured values like lists or maps are accepted as
* static op template values. A static value is any value which contains zero bind points at any level.
*
*
Dynamic Op Fields with Binding References
- * Example:
* {@code
* op:
* field1: "{binding1}"
@@ -247,34 +245,32 @@ import java.util.function.LongFunction;
* null values invalid for ANY op template value.
*
* Dynamic Op Fields with Binding Definitions
- * Example:
- *
* {@code
* op:
- * field1: "{{NumberNameToString()}}"
- * field2: "value is: {{NumberNameToString()}}"
+ * field1: "{{NumberNameToString()}}"
+ * field2: "value is: {{NumberNameToString()}}"
* }
*
*
* This form has exactly the same effect as the previous example as long as your bindings definitions included:
*
{@code
* bindings:
- * binding1: NumberNameToString();
+ * binding1: NumberNameToString();
* }
*
* Dynamic Op Fields with Structure
- * Example:
*
* {@code
- * field1:
- * k1: "{binding1}
- * k2: "literal value"
- * field2:
- * - "value3"
- * - "{binding4}"
- * - "a value: {binding5}"
- * - "{{NumberNameToString}}"
- * - "a value: {{NumberNameToString()}}"
+ * op:
+ * field1:
+ * k1: "{binding1}
+ * k2: "literal value"
+ * field2:
+ * - "value3"
+ * - "{binding4}"
+ * - "a value: {binding5}"
+ * - "{{NumberNameToString}}"
+ * - "a value: {{NumberNameToString()}}"
* }
*
* This example combines the previous ones with structure and dynamic values. Both field1 and field2 are dynamic,
@@ -286,15 +282,14 @@ import java.util.function.LongFunction;
* configure your binding definitions thusly.
*
* Op Template Params
- * Example:
* {@code
* params:
- * prepared: true
+ * prepared: true
* ops:
- * op1:
- * field1: value1
- * params:
- * prepared: false
+ * op1:
+ * field1: value1
+ * params:
+ * prepared: false
* }
*
* The params section are the first layer of external configuration values that an op template can use to distinguish
@@ -308,7 +303,6 @@ import java.util.function.LongFunction;
* down to each block and then down to each statement.
*
*
Activity Params
- * Example:
* {@code
* ./nb run driver=... workload=... cl=LOCAL_QUORUM
* }
@@ -317,6 +311,19 @@ import java.util.function.LongFunction;
* another fallback source for configuration parameters. The {@link ParsedOp} implementation will automatically look
* in the activity parameters if needed to find a missing configuration parameter, but this will only work if
* the specific named parameter is allowed at the activity level.
+ *
+ *
+ * Alternate Names
+ * Sometimes you may need to support more than one name for the same purpose. In such cases, there are helper
+ * methods which can be used to reduce from a set of possible field names to a single one.
+ *
+ * - {@link #requiredFieldOf(String...)} and {@link #requiredFieldOf(List)} will find exactly one field within the set of
+ * possible fields.
+ * Zero or more than one will throw an error.
+ * - {@link #optionalFieldOf(String...)} and {@link #optionalFieldOf(List)} will find exactly zero or one
+ * field within the set of possible fields. More than one will throw an error.
+ *
+ *
*/
public class ParsedOp extends NBBaseComponent implements LongFunction