mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
Merge remote-tracking branch 'origin/main' into driver/weaviate
This commit is contained in:
Madhavan Sridharan
commit
6fc4b14f15
6
.github/workflows/preview.yml
vendored
6
.github/workflows/preview.yml
vendored
@ -82,7 +82,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: docker test build
|
||||
uses: docker/build-push-action@v6.2.0
|
||||
uses: docker/build-push-action@v6.5.0
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
@ -133,7 +133,7 @@ jobs:
|
||||
scripts/bump-minor-version
|
||||
|
||||
- name: docker push to hub
|
||||
uses: docker/build-push-action@v6.2.0
|
||||
uses: docker/build-push-action@v6.5.0
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
@ -144,7 +144,7 @@ jobs:
|
||||
|
||||
# https://github.com/softprops/action-gh-release
|
||||
- name: create github release
|
||||
uses: softprops/action-gh-release@v2.0.6
|
||||
uses: softprops/action-gh-release@v2.0.8
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
# body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
|
||||
6
.github/workflows/release.yml
vendored
6
.github/workflows/release.yml
vendored
@ -77,7 +77,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: docker test build
|
||||
uses: docker/build-push-action@v6.2.0
|
||||
uses: docker/build-push-action@v6.5.0
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
@ -118,7 +118,7 @@ jobs:
|
||||
scripts/bump-minor-version
|
||||
|
||||
- name: docker push to hub
|
||||
uses: docker/build-push-action@v6.2.0
|
||||
uses: docker/build-push-action@v6.5.0
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
@ -129,7 +129,7 @@ jobs:
|
||||
|
||||
# https://github.com/softprops/action-gh-release
|
||||
- name: create github release
|
||||
uses: softprops/action-gh-release@v2.0.6
|
||||
uses: softprops/action-gh-release@v2.0.8
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
# body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
|
||||
@ -26,7 +26,7 @@
|
||||
|
||||
<properties>
|
||||
|
||||
<revision>5.21.2-SNAPSHOT</revision>
|
||||
<revision>5.21.3-SNAPSHOT</revision>
|
||||
<!-- Set this level to override the logging level for tests during build -->
|
||||
<project.testlevel>INFO</project.testlevel>
|
||||
<!-- Set this level to override the logging level for tests logging configuration during build -->
|
||||
@ -645,6 +645,7 @@
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<argLine>--enable-preview @{argLine}</argLine>
|
||||
<forkCount>1</forkCount>
|
||||
<reuseForks>false</reuseForks>
|
||||
<includes>
|
||||
@ -909,6 +910,7 @@
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<configuration>
|
||||
<additionalJOptions>--enable-preview</additionalJOptions>
|
||||
<release>21</release>
|
||||
<doctitle>${javadoc.name}</doctitle>
|
||||
<windowtitle>${javadoc.name}</windowtitle>
|
||||
@ -917,6 +919,7 @@
|
||||
<detectOfflineLinks>false</detectOfflineLinks>
|
||||
<!-- <additionalparam>-Xdoclint:none</additionalparam>-->
|
||||
<additionalOptions>
|
||||
|
||||
<additionalOption>-Xdoclint:none</additionalOption>
|
||||
</additionalOptions>
|
||||
<!-- <additionalJOption>-Xdoclint:none</additionalJOption>-->
|
||||
|
||||
@ -16,16 +16,16 @@
|
||||
|
||||
package io.nosqlbench.adapter.http;
|
||||
|
||||
import io.nosqlbench.adapter.http.core.HttpFormatParser;
|
||||
import io.nosqlbench.adapter.http.core.HttpOp;
|
||||
import io.nosqlbench.adapter.http.core.HttpOpMapper;
|
||||
import io.nosqlbench.adapter.http.core.HttpSpace;
|
||||
import io.nosqlbench.adapter.http.core.*;
|
||||
import io.nosqlbench.nb.api.components.core.NBComponentProps;
|
||||
import io.nosqlbench.nb.api.config.standard.ConfigModel;
|
||||
import io.nosqlbench.nb.api.config.standard.Param;
|
||||
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram;
|
||||
import io.nosqlbench.nb.api.labels.NBLabels;
|
||||
import io.nosqlbench.nb.api.components.core.NBComponent;
|
||||
import io.nosqlbench.nb.annotations.Service;
|
||||
@ -41,8 +41,16 @@ import java.util.function.Function;
|
||||
@Service(value = DriverAdapter.class, selector = "http")
|
||||
public class HttpDriverAdapter extends BaseDriverAdapter<HttpOp, HttpSpace> {
|
||||
|
||||
public final NBMetricHistogram statusCodeHistogram;
|
||||
|
||||
public HttpDriverAdapter(NBComponent parent, NBLabels labels) {
|
||||
super(parent, labels);
|
||||
this.statusCodeHistogram = create().histogram(
|
||||
"statuscode",
|
||||
Integer.parseInt(getComponentProp(NBComponentProps.HDRDIGITS).orElse("3")),
|
||||
MetricCategory.Payload,
|
||||
"A histogram of status codes received by the HTTP client"
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -54,7 +62,7 @@ public class HttpDriverAdapter extends BaseDriverAdapter<HttpOp, HttpSpace> {
|
||||
|
||||
@Override
|
||||
public Function<String, ? extends HttpSpace> getSpaceInitializer(NBConfiguration cfg) {
|
||||
return spaceName -> new HttpSpace(getParent(), spaceName, cfg);
|
||||
return spaceName -> new HttpSpace(this, spaceName, cfg);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -83,5 +91,4 @@ public class HttpDriverAdapter extends BaseDriverAdapter<HttpOp, HttpSpace> {
|
||||
|
||||
return super.getConfigModel().add(HttpSpace.getConfigModel()).add(thisCfgModel);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.adapter.http.core;
|
||||
|
||||
import com.codahale.metrics.Histogram;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
|
||||
import io.nosqlbench.nb.api.labels.NBLabeledElement;
|
||||
import io.nosqlbench.nb.api.labels.NBLabels;
|
||||
import io.nosqlbench.nb.api.components.core.NBComponent;
|
||||
|
||||
public class HttpMetrics implements NBLabeledElement {
|
||||
private final NBComponent parent;
|
||||
private final HttpSpace space;
|
||||
final Histogram statusCodeHistogram;
|
||||
|
||||
public HttpMetrics(NBComponent parent, HttpSpace space) {
|
||||
this.parent = parent;
|
||||
this.space = space;
|
||||
statusCodeHistogram = parent.create().histogram(
|
||||
"statuscode",
|
||||
space.getHdrDigits(),
|
||||
MetricCategory.Payload,
|
||||
"A histogram of status codes received by the HTTP client"
|
||||
);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return "http"+("default".equals(this.space.getSpaceName())?"": '-' + space.getSpaceName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public NBLabels getLabels() {
|
||||
return space.getLabels();
|
||||
}
|
||||
}
|
||||
@ -19,24 +19,13 @@ package io.nosqlbench.adapter.http.core;
|
||||
import io.nosqlbench.adapter.http.errors.InvalidResponseBodyException;
|
||||
import io.nosqlbench.adapter.http.errors.InvalidStatusCodeException;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.RunnableOp;
|
||||
import org.apache.logging.log4j.core.tools.picocli.CommandLine;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Spliterators;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
public class HttpOp implements CycleOp {
|
||||
|
||||
@ -75,7 +64,7 @@ public class HttpOp implements CycleOp {
|
||||
try {
|
||||
CompletableFuture<HttpResponse<String>> responseFuture = client.sendAsync(request, bodyreader);
|
||||
response = responseFuture.get(space.getTimeoutMillis(), TimeUnit.MILLISECONDS);
|
||||
space.getHttpMetrics().statusCodeHistogram.update(response.statusCode());
|
||||
space.statusCodeHistogram.update(response.statusCode());
|
||||
|
||||
if (ok_status != null) {
|
||||
if (!ok_status.matcher(String.valueOf(response.statusCode())).matches()) {
|
||||
|
||||
@ -16,6 +16,9 @@
|
||||
|
||||
package io.nosqlbench.adapter.http.core;
|
||||
|
||||
import io.nosqlbench.adapter.http.HttpDriverAdapter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricHistogram;
|
||||
import io.nosqlbench.nb.api.labels.NBLabeledElement;
|
||||
import io.nosqlbench.nb.api.labels.NBLabels;
|
||||
import io.nosqlbench.nb.api.config.standard.ConfigModel;
|
||||
@ -39,24 +42,25 @@ import java.util.Locale;
|
||||
public class HttpSpace implements NBLabeledElement {
|
||||
private final static Logger logger = LogManager.getLogger(HttpSpace.class);
|
||||
|
||||
private final NBComponent parent;
|
||||
private final HttpDriverAdapter parentAdapter;
|
||||
private final String name;
|
||||
private final NBConfiguration cfg;
|
||||
public NBMetricHistogram statusCodeHistogram;
|
||||
private HttpConsoleFormats console;
|
||||
private HttpClient.Redirect followRedirects;
|
||||
private Duration timeout;
|
||||
private long timeoutMillis;
|
||||
private final HttpClient httpclient;
|
||||
private int hdrDigits;
|
||||
private HttpMetrics httpMetrics;
|
||||
private boolean diagnosticsEnabled;
|
||||
|
||||
|
||||
public HttpSpace(NBComponent parent, String spaceName, NBConfiguration cfg) {
|
||||
this.parent = parent;
|
||||
public HttpSpace(HttpDriverAdapter parentAdapter, String spaceName, NBConfiguration cfg) {
|
||||
this.parentAdapter = parentAdapter;
|
||||
this.name = spaceName;
|
||||
this.cfg = cfg;
|
||||
applyConfig(cfg);
|
||||
this.statusCodeHistogram = parentAdapter.statusCodeHistogram;
|
||||
this.httpclient = newClient();
|
||||
}
|
||||
|
||||
@ -79,10 +83,9 @@ public class HttpSpace implements NBLabeledElement {
|
||||
);
|
||||
this.timeout = Duration.ofMillis(cfg.get("timeout", long.class));
|
||||
this.timeoutMillis = cfg.get("timeout", long.class);
|
||||
this.httpMetrics = new HttpMetrics(parent, this);
|
||||
|
||||
this.console = cfg.getOptional("diag").map(s -> HttpConsoleFormats.apply(s, this.console))
|
||||
.orElseGet(() -> HttpConsoleFormats.apply(null,null));
|
||||
.orElseGet(() -> HttpConsoleFormats.apply(null, null));
|
||||
|
||||
this.diagnosticsEnabled = console.isDiagnosticMode();
|
||||
|
||||
@ -105,10 +108,6 @@ public class HttpSpace implements NBLabeledElement {
|
||||
return name;
|
||||
}
|
||||
|
||||
public HttpMetrics getHttpMetrics() {
|
||||
return httpMetrics;
|
||||
}
|
||||
|
||||
public boolean isDiagnosticMode() {
|
||||
return diagnosticsEnabled;
|
||||
}
|
||||
@ -124,18 +123,15 @@ public class HttpSpace implements NBLabeledElement {
|
||||
.setDescription("Whether to follow redirects. Normal redirects are those which do not " +
|
||||
"redirect from HTTPS to HTTP.")
|
||||
)
|
||||
.add(Param.optional(List.of("diag","diagnostics"), String.class)
|
||||
.add(Param.optional(List.of("diag", "diagnostics"), String.class)
|
||||
.setDescription("Print extended diagnostics. This option has numerous" +
|
||||
" possible values. See the markdown docs for details. (nb help http)")
|
||||
)
|
||||
.add(Param.defaultTo("timeout", 1000L*60L*15L) // 15 minutes
|
||||
.add(Param.defaultTo("timeout", 1000L * 60L * 15L) // 15 minutes
|
||||
.setDescription("How long to wait for requests before timeout out. Default is forever."))
|
||||
.add(Param.defaultTo("hdr_digits", 4)
|
||||
.setDescription("number of digits of precision to keep in HDR histograms"))
|
||||
.asReadOnly();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -70,7 +70,7 @@ import java.util.function.LongFunction;
|
||||
* specification tests under the workload_definition folder of the adapters-api module.
|
||||
* </P>
|
||||
*
|
||||
*
|
||||
* <HR/>
|
||||
* <H2>Op Template Parsing</H2>
|
||||
* <OL>
|
||||
* <LI>Rule #1: All op templates are parsed into an internal normalized structure which contains:
|
||||
@ -92,36 +92,41 @@ import java.util.function.LongFunction;
|
||||
* The net effect of these rules is that the NoSQLBench driver developer may safely use functional forms to access data
|
||||
* in the op template, or may decide that certain op fields must only be provided in a static way per operation.
|
||||
* </P>
|
||||
*
|
||||
* <HR/>
|
||||
* <H2>Distinguishing Op Payload from Op Config</H2>
|
||||
* <P>When a user specifies an op template, they may choose to provide only a single set of op fields without
|
||||
* distinguishing between config or payload, or they may choose to directly configure each.
|
||||
* <H4>Example:</H4>
|
||||
*
|
||||
* <PRE>{@code
|
||||
* ops:
|
||||
* # both op and params explicitly named
|
||||
* op1:
|
||||
* op:
|
||||
* opfield1: value1
|
||||
* params:
|
||||
* param2: value2
|
||||
* # neither op field nor params named, so all assumed to be op fields
|
||||
* op2:
|
||||
* opfield1: value1
|
||||
* param2: value2
|
||||
* # in this case, if param2 is meant to be config level,
|
||||
* # it is a likely config error that should be thrown to the user
|
||||
* # only op field explicitly named, so remainder automatically pushed into params
|
||||
* op3:
|
||||
* op:
|
||||
* opfield1: value1
|
||||
* param2: value2
|
||||
* # only params explicitly named, so remainder pushed into op payload
|
||||
* op4:
|
||||
* params:
|
||||
* param2: value2
|
||||
* opfield1: value1
|
||||
* }</PRE>
|
||||
*
|
||||
* # both op and params explicitly named
|
||||
* op1:
|
||||
* op:
|
||||
* opfield1: value1
|
||||
* params:
|
||||
* param2: value2
|
||||
*
|
||||
* # neither op field nor params named, so all assumed to be op fields
|
||||
* op2:
|
||||
* opfield1: value1
|
||||
* param2: value2
|
||||
*
|
||||
* # in this case, if param2 is meant to be config level,
|
||||
* # it is a likely config error that should be thrown to the user
|
||||
* # only op field explicitly named, so remainder automatically pushed into params
|
||||
* op3:
|
||||
* op:
|
||||
* opfield1: value1
|
||||
* param2: value2
|
||||
*
|
||||
* # only params explicitly named, so remainder pushed into op payload
|
||||
* op4:
|
||||
* params:
|
||||
* param2: value2
|
||||
* opfield1: value1
|
||||
* }</PRE></P>
|
||||
*
|
||||
* <p>
|
||||
* All of these are considered valid constructions, and all of them may actually achieve the same result.
|
||||
* This looks like an undesirable problem, but it serves to simplify things for users in one specific way: It allows
|
||||
@ -135,37 +140,37 @@ import java.util.function.LongFunction;
|
||||
* <HR></HR>
|
||||
* <H2>Design Invariants</H2>
|
||||
*
|
||||
* <P>The above rules imply invariants, which are made explicit here. {@link ParsedOp}.</P>
|
||||
* <P>The above rules imply invariants, which are made explicit here.</P>
|
||||
*
|
||||
* <P><UL>
|
||||
* <H4>Single Purpose Fields</H4>
|
||||
* <P><EM>You may not use an op field name or parameter name for more than one purpose.</EM></P>
|
||||
*
|
||||
* <LI><EM>You may not use an op field name or parameter name for more than one purpose.</EM>
|
||||
* <UL>
|
||||
* <LI>Treat all parameters supported by a driver adapter and it's op fields as a globally shared namespace, even if it
|
||||
* is not.
|
||||
* This avoids creating any confusion about what a parameter can be used for and how to use it for the right thing in
|
||||
* the right place.
|
||||
* For example, you may not use the parameter name `socket` in an op template to mean one thing and then use it
|
||||
* <H4>Shared Namespace</H4>
|
||||
* <P><EM>Treat all parameters supported by a driver adapter and it's op fields as a globally shared namespace, even
|
||||
* if it is not.</EM></P>
|
||||
*
|
||||
* <P>This avoids creating any confusion about what a parameter can be used for and how to use it for the right thing in
|
||||
* the right place. For example, you may not use the parameter name `socket` in an op template to mean one thing and then use it
|
||||
* at the driver adapter level to mean something different. However, if the meaning is congruent, a driver developer
|
||||
* may choose to support some cross-cutting parameters at the activity level. These allowances are explicit,
|
||||
* however, as each driver dictates what it will allow as activity parameters.
|
||||
* </LI>
|
||||
* </UL>
|
||||
* </LI>
|
||||
* however, as each driver dictates what it will allow as activity parameters.</P>
|
||||
*
|
||||
* <H3>Layered Resolution</H3>
|
||||
*
|
||||
* <P><EM>Users may specify op payload fields within op params or activity params as fallback config sources in that
|
||||
* order.</EM></P>
|
||||
*
|
||||
* <LI><EM>Users may specify op payload fields within op params or activity params as fallback config sources in that
|
||||
* order.</EM>
|
||||
* <UL>
|
||||
* <LI>IF a name is valid as an op field, it must also be valid as such when specified in op params.</LI>
|
||||
* <LI>If a name is valid as an op field, it must also be valid as such when specified in activity params, within the
|
||||
* scope of {@link ParsedOp}</LI>
|
||||
* <LI>When an op field is found via op params or activity params, it may NOT be dynamic. If dynamic values are intended
|
||||
* to be provided
|
||||
* at a common layer in the workload, then bindings support this already.</LI>
|
||||
* to be provided at a common layer in the workload, then bindings or template variables support this already.</LI>
|
||||
* </UL>
|
||||
* </LI>
|
||||
*
|
||||
* <LI><EM>You must access non-payload params via Config-oriented methods.</EM>
|
||||
* <H3>Configuration Fields</H3>
|
||||
* <EM>You must access non-payload params via Config-oriented methods.</EM>
|
||||
* <UL>
|
||||
* <LI>Op Templates contain op payload data and op configs (params, activity params).</LI>
|
||||
* <LI>You must use only {@link ParsedOp} getters with "...Config..." names, such as
|
||||
@ -176,56 +181,49 @@ import java.util.function.LongFunction;
|
||||
* </UL>
|
||||
* </LI>
|
||||
*
|
||||
* <LI><EM>The user must be warned when a required or optional config value is missing from op params (or activity
|
||||
* params), but a value
|
||||
* of the same name is found in op payload fields.</EM>
|
||||
* <UL>
|
||||
* <LI>If rule #1 is followed, and names are unambiguous across the driver, then it is almost certainly a configuration
|
||||
* error.</LI>
|
||||
* </UL>
|
||||
* </LI>
|
||||
* <H3>Sanity Checks</H3>
|
||||
*
|
||||
* <LI><EM>When both an op payload field and a param field of the same name are defined through cascading configuration
|
||||
* of param fields,
|
||||
* the local op payload field takes precedence.</EM>
|
||||
* <P><EM>The user must be warned when a required or optional config value is missing from op params (or activity
|
||||
* params), but a value of the same name is found in op payload fields.</EM>
|
||||
* If rule #1 is followed, and names are unambiguous across the driver, then it is almost certainly a configuration
|
||||
* error.</P>
|
||||
*
|
||||
* <H3>Precedence</H3>
|
||||
* <P>The order of precedence for values is:
|
||||
* <UL>
|
||||
* <LI>This is an extension of the param override rules which say that the closest (most local) value to an operation is
|
||||
* the one that takes precedence.</LI>
|
||||
* <LI>In practice, there will be no conflicts between direct static and dynamic fields, but there will be possibly
|
||||
* between
|
||||
* static or dynamic fields and parameters and activity params. If a user wants to promote an activity param as an
|
||||
* override to existing op fields,
|
||||
* template variables allow for this to happen gracefully. Otherwise, the order of precedence is 1) op fields 2) op
|
||||
* params 3) activity params.</LI>
|
||||
* </UL>
|
||||
* </LI>
|
||||
* <LI>op payload fields</LI>
|
||||
* <LI>op param fields</LI>
|
||||
* <LI>activity params (when enabled, see below)</LI>
|
||||
* </UL>
|
||||
* </P>
|
||||
*
|
||||
* <H3>Enabling Activity Params</H3>
|
||||
* <P>If a user wants to allow an activity param as an default for an fields, they must publish the op field
|
||||
* name in the configuration model for the activity. Otherwise it is an error to specify the value at the activity
|
||||
* level.</P>
|
||||
*
|
||||
* <HR></HR>
|
||||
*
|
||||
* <H2>Op Payload Forms</H2>
|
||||
* Field values can come from multiple sources. These forms and any of their combinations are supported.
|
||||
*
|
||||
* <H3>Static Op Fields</H3>
|
||||
* <H4>Example:</H4>
|
||||
* <PRE>{@code
|
||||
* op:
|
||||
* field1: value1
|
||||
* field2:
|
||||
* map3:
|
||||
* key4: value4
|
||||
* map5:
|
||||
* key6: value6
|
||||
* field7: false
|
||||
* field8: 8.8
|
||||
* field1: value1
|
||||
* field2:
|
||||
* map3:
|
||||
* key4: value4
|
||||
* map5:
|
||||
* key6: value6
|
||||
* field7: false
|
||||
* field8: 8.8
|
||||
* }</PRE>
|
||||
* <p>
|
||||
* As shown, any literal value of any valid YAML type, including structured values like lists or maps are accepted as
|
||||
* static op template values. A static value is any value which contains zero bind points at any level.
|
||||
*
|
||||
* <H3>Dynamic Op Fields with Binding References</H3>
|
||||
* <H4>Example:</H4>
|
||||
* <PRE>{@code
|
||||
* op:
|
||||
* field1: "{binding1}"
|
||||
@ -247,34 +245,32 @@ import java.util.function.LongFunction;
|
||||
* null values invalid for ANY op template value.</P>
|
||||
*
|
||||
* <H3>Dynamic Op Fields with Binding Definitions</H3>
|
||||
* <H4>Example:</H4>
|
||||
*
|
||||
* <PRE>{@code
|
||||
* op:
|
||||
* field1: "{{NumberNameToString()}}"
|
||||
* field2: "value is: {{NumberNameToString()}}"
|
||||
* field1: "{{NumberNameToString()}}"
|
||||
* field2: "value is: {{NumberNameToString()}}"
|
||||
* }
|
||||
* </PRE>
|
||||
* <p>
|
||||
* This form has exactly the same effect as the previous example as long as your bindings definitions included:
|
||||
* <PRE>{@code
|
||||
* bindings:
|
||||
* binding1: NumberNameToString();
|
||||
* binding1: NumberNameToString();
|
||||
* }</PRE>
|
||||
*
|
||||
* <H3>Dynamic Op Fields with Structure</H3>
|
||||
* <H4>Example:</H4>
|
||||
*
|
||||
* <PRE>{@code
|
||||
* field1:
|
||||
* k1: "{binding1}
|
||||
* k2: "literal value"
|
||||
* field2:
|
||||
* - "value3"
|
||||
* - "{binding4}"
|
||||
* - "a value: {binding5}"
|
||||
* - "{{NumberNameToString}}"
|
||||
* - "a value: {{NumberNameToString()}}"
|
||||
* op:
|
||||
* field1:
|
||||
* k1: "{binding1}
|
||||
* k2: "literal value"
|
||||
* field2:
|
||||
* - "value3"
|
||||
* - "{binding4}"
|
||||
* - "a value: {binding5}"
|
||||
* - "{{NumberNameToString}}"
|
||||
* - "a value: {{NumberNameToString()}}"
|
||||
* }</PRE>
|
||||
*
|
||||
* <P>This example combines the previous ones with structure and dynamic values. Both field1 and field2 are dynamic,
|
||||
@ -286,15 +282,14 @@ import java.util.function.LongFunction;
|
||||
* configure your binding definitions thusly.</P>
|
||||
*
|
||||
* <H3>Op Template Params</H3>
|
||||
* <H4>Example:</H4>
|
||||
* <PRE>{@code
|
||||
* params:
|
||||
* prepared: true
|
||||
* prepared: true
|
||||
* ops:
|
||||
* op1:
|
||||
* field1: value1
|
||||
* params:
|
||||
* prepared: false
|
||||
* op1:
|
||||
* field1: value1
|
||||
* params:
|
||||
* prepared: false
|
||||
* }</PRE>
|
||||
* <p>
|
||||
* The params section are the first layer of external configuration values that an op template can use to distinguish
|
||||
@ -308,7 +303,6 @@ import java.util.function.LongFunction;
|
||||
* down to each block and then down to each statement.
|
||||
*
|
||||
* <H3>Activity Params</H3>
|
||||
* <H4>Example:</H4>
|
||||
* <PRE>{@code
|
||||
* ./nb run driver=... workload=... cl=LOCAL_QUORUM
|
||||
* }</PRE>
|
||||
@ -317,6 +311,19 @@ import java.util.function.LongFunction;
|
||||
* another fallback source for configuration parameters. The {@link ParsedOp} implementation will automatically look
|
||||
* in the activity parameters if needed to find a missing configuration parameter, but this will only work if
|
||||
* the specific named parameter is allowed at the activity level.</P>
|
||||
*
|
||||
* <HR/>
|
||||
* <H2>Alternate Names</H2>
|
||||
* <P>Sometimes you may need to support more than one name for the same purpose. In such cases, there are helper
|
||||
* methods which can be used to reduce from a set of possible field names to a single one.
|
||||
* <UL>
|
||||
* <LI>{@link #requiredFieldOf(String...)} and {@link #requiredFieldOf(List)} will find exactly one field within the set of
|
||||
* possible fields.
|
||||
* Zero or more than one will throw an error.</LI>
|
||||
* <LI>{@link #optionalFieldOf(String...)} and {@link #optionalFieldOf(List)} will find exactly zero or one
|
||||
* field within the set of possible fields. More than one will throw an error.</LI>
|
||||
* </UL>
|
||||
* </P>
|
||||
*/
|
||||
public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String, ?>>, NBComponent, StaticFieldReader, DynamicFieldReader {
|
||||
|
||||
@ -571,15 +578,6 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
|
||||
}
|
||||
|
||||
|
||||
public <T> Optional<T> getOptionalStaticValue(List<String> fieldNames, Class<T> classOfT) {
|
||||
for (String field : fieldNames) {
|
||||
if (isStatic(field)) {
|
||||
return Optional.ofNullable(tmap.getStaticValue(field, classOfT));
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
public <T> Optional<T> takeOptionalStaticValue(String field, Class<T> classOfT) {
|
||||
return Optional.ofNullable(tmap.takeStaticValue(field, classOfT));
|
||||
}
|
||||
@ -919,24 +917,6 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
|
||||
return lfa;
|
||||
}
|
||||
|
||||
public <FA, FE> LongFunction<FA> enhanceFunc(
|
||||
LongFunction<FA> func,
|
||||
List<String> fields,
|
||||
Class<FE> type,
|
||||
BiFunction<FA, FE, FA> combiner
|
||||
) {
|
||||
for (String field : fields) {
|
||||
if (isDefined(field)) {
|
||||
LongFunction<FE> fieldEnhancerFunc = getAsRequiredFunction(field, type);
|
||||
LongFunction<FA> lfa = l -> combiner.apply(func.apply(l), fieldEnhancerFunc.apply(l));
|
||||
return lfa;
|
||||
}
|
||||
}
|
||||
throw new OpConfigError("One of the op fields from " + fields + " is required, but none was found " +
|
||||
"in the op template named '" + this.getName() + "'");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* <p>Enhance a {@link Function} with a named optional function IFF it exists.</p>
|
||||
*
|
||||
@ -969,23 +949,6 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
|
||||
return lfa;
|
||||
}
|
||||
|
||||
public <FA, FE> LongFunction<FA> enhanceFuncOptionally(
|
||||
LongFunction<FA> func,
|
||||
List<String> fields,
|
||||
Class<FE> type,
|
||||
BiFunction<FA, FE, FA> combiner
|
||||
) {
|
||||
for (String field : fields) {
|
||||
Optional<LongFunction<FE>> fieldEnhancerFunc = getAsOptionalFunction(field, type);
|
||||
if (fieldEnhancerFunc.isPresent()) {
|
||||
LongFunction<FE> feLongFunction = fieldEnhancerFunc.get();
|
||||
LongFunction<FA> lfa = l -> combiner.apply(func.apply(l), feLongFunction.apply(l));
|
||||
return lfa;
|
||||
}
|
||||
}
|
||||
return func;
|
||||
}
|
||||
|
||||
|
||||
public static enum SubOpNaming {
|
||||
SubKey,
|
||||
@ -1241,4 +1204,38 @@ public class ParsedOp extends NBBaseComponent implements LongFunction<Map<String
|
||||
}
|
||||
|
||||
|
||||
|
||||
private String oneFieldOrNull(List<String> fields) {
|
||||
int count = 0;
|
||||
String found = null;
|
||||
for (String field : fields) {
|
||||
if (isDefined(field)) {
|
||||
if (found == null) {
|
||||
found = field;
|
||||
} else {
|
||||
throw new OpConfigError(STR."exactly one of \{fields} was required, but it was found as both \{found} and \{field}");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
public Optional<String> optionalFieldOf(List<String> fields) {
|
||||
return Optional.ofNullable(oneFieldOrNull(fields));
|
||||
}
|
||||
public Optional<String> optionalFieldOf(String... fields) {
|
||||
return Optional.ofNullable(oneFieldOrNull(List.of(fields)));
|
||||
}
|
||||
public String requiredFieldOf(String... fields) {
|
||||
return requiredFieldOf(List.of(fields));
|
||||
}
|
||||
public String requiredFieldOf(List<String> fields) {
|
||||
String field = oneFieldOrNull(fields);
|
||||
if (field==null) {
|
||||
throw new OpConfigError(STR."no fields were found with name in (\{fields}) for op template \{this}");
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -38,7 +38,8 @@ public class NBBaseComponentMetrics implements NBComponentMetrics {
|
||||
lock.lock();
|
||||
String openMetricsName = metric.getLabels().linearizeAsMetrics();
|
||||
if (metrics.containsKey(openMetricsName)) {
|
||||
throw new RuntimeException("Can't add the same metric by label set to the same live component:" + openMetricsName);
|
||||
throw new RuntimeException("Can't add the same metric by label set to the same live component:" +
|
||||
" this:" + this.toString());
|
||||
}
|
||||
metrics.put(openMetricsName,metric);
|
||||
for (MetricRegistryListener listener : listeners) {
|
||||
|
||||
@ -20,6 +20,7 @@ import java.util.Optional;
|
||||
|
||||
public interface NBComponentProps {
|
||||
String SUMMARY = "summary";
|
||||
String HDRDIGITS = "hdr_digits";
|
||||
|
||||
Optional<String> getComponentProp(String name);
|
||||
NBComponentProps setComponentProp(String name, String value);
|
||||
|
||||
@ -142,6 +142,7 @@
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<argLine>--enable-preview @{argLine}</argLine>
|
||||
<groups>perf</groups>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
@ -56,6 +56,7 @@
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<argLine>--enable-preview @{argLine}</argLine>
|
||||
<additionalClasspathElements>
|
||||
<additionalClasspathElement>${project.basedir}/target/virtdata-userlibs-${project.version}.jar
|
||||
</additionalClasspathElement>
|
||||
|
||||
@ -302,7 +302,7 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
|
||||
} else if (isConfig(field)) {
|
||||
return getConfig(field);
|
||||
}
|
||||
logger.warn("static field '{}' was requested, but it does not exist", field);
|
||||
// logger.warn("static field '{}' was requested, but it does not exist", field);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@ -94,6 +94,7 @@
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<argLine>--enable-preview @{argLine}</argLine>
|
||||
<forkCount>1</forkCount>
|
||||
<reuseForks>false</reuseForks>
|
||||
<includes>
|
||||
|
||||
@ -75,6 +75,7 @@
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<argLine>--enable-preview @{argLine}</argLine>
|
||||
<parallel>methods</parallel>
|
||||
<!-- <additionalClasspathElements>-->
|
||||
<!-- <additionalClasspathElement>${project.basedir}/target/virtdata-userlibs-${project.version}.jar-->
|
||||
|
||||
@ -29,12 +29,13 @@ class ExitStatusIntegrationTests {
|
||||
|
||||
private final static String JARNAME = "target/nbr.jar";
|
||||
|
||||
|
||||
@Test
|
||||
void testExitStatusOnBadParam() {
|
||||
ProcessInvoker invoker = new ProcessInvoker();
|
||||
invoker.setLogDir("logs/test");
|
||||
ProcessResult result = invoker.run("exitstatus_badparam", 15,
|
||||
java, "-jar", JARNAME, "--logs-dir", "logs/test/badparam/",
|
||||
java, "--enable-preview", "-jar", JARNAME, "--logs-dir", "logs/test/badparam/",
|
||||
"badparam"
|
||||
);
|
||||
assertThat(result.exception).isNull();
|
||||
@ -48,7 +49,7 @@ class ExitStatusIntegrationTests {
|
||||
ProcessInvoker invoker = new ProcessInvoker();
|
||||
invoker.setLogDir("logs/test");
|
||||
ProcessResult result = invoker.run("exitstatus_initexception", 15,
|
||||
java, "-jar", JARNAME, "--logs-dir", "logs/test/initerror", "run",
|
||||
java, "--enable-preview", "-jar", JARNAME, "--logs-dir", "logs/test/initerror", "run",
|
||||
"driver=diag", "op=initdelay:initdelay=notanumber"
|
||||
);
|
||||
assertThat(result.exception).isNull();
|
||||
@ -64,7 +65,8 @@ class ExitStatusIntegrationTests {
|
||||
|
||||
// Forcing a thread exception via basic command issue.
|
||||
ProcessResult result = invoker.run("exitstatus_threadexception", 30,
|
||||
"java", "-jar", JARNAME, "--logs-dir", "logs/test/threadexcep", "--logs-level", "debug", "run",
|
||||
"java", "--enable-preview", "-jar", JARNAME, "--logs-dir", "logs/test/threadexcep", "--logs-level",
|
||||
"debug", "run",
|
||||
"driver=diag", "cyclerate=10", "not_a_thing", "cycles=100", "-vvv"
|
||||
);
|
||||
String stdout = String.join("\n", result.getStdoutData());
|
||||
@ -77,7 +79,7 @@ class ExitStatusIntegrationTests {
|
||||
ProcessInvoker invoker = new ProcessInvoker();
|
||||
invoker.setLogDir("logs/test");
|
||||
ProcessResult result = invoker.run("exitstatus_asyncstoprequest", 60,
|
||||
"java", "-jar", JARNAME, "--logs-dir", "logs/test/asyncstop", "--logs-level", "debug", "run",
|
||||
"java", "--enable-preview", "-jar", JARNAME, "--logs-dir", "logs/test/asyncstop", "--logs-level", "debug", "run",
|
||||
"driver=diag", "threads=2", "cyclerate=10", "op=erroroncycle:erroroncycle=10", "cycles=50", "-vvv"
|
||||
);
|
||||
assertThat(result.exception).isNull();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user