diff --git a/devdocs/devguide/driver_standards.md b/devdocs/devguide/driver_standards.md index 455a06efc..22d8530dc 100644 --- a/devdocs/devguide/driver_standards.md +++ b/devdocs/devguide/driver_standards.md @@ -2,6 +2,7 @@ This is a work in progress... + This is the document to read if you want to know if your NoSQLBench driver is complete. Within this document, the phrase `conformant` will be taken to mean that a driver or feature is implemented according to the design intent and standards of the NoSQLBench driver API. @@ -13,6 +14,22 @@ reliable way for users from one driver to another. Over time, the standards in this guide will be programmatically enforced by the NoSQLBench driver API. +## Op Templates + +The core building block of a NoSQLBench driver is the op template. This is the form of a +statement or operation that users add to a yaml or workload editor to represent a single operation. + +For example, in the CQL driver, this is called a "statement template", but going forward, they will +all be called Op Templates and internal API names will reflect that. + +It is the driver's responsibility to create a quick-draw version of an operation. + +## Op Sequencing + +A conformant driver should use the standard method of creating an operational sequence. This means +that a driver simply has to provide a function to map an OpTemplate to a more ready to use form that +is specific to the low level driver in question. + ## Terms - NB Driver - The NoSQLBench level driver, the code that this document @@ -120,8 +137,17 @@ The next exception handler `ignore` should also be called, but this is simply a named 'no-op' which is generally the last fall-through case in a switch statement. +TBD + +## Result Validation + +## Diagnostic Mode + + ## Naming Conventions +TBD + ### Parameter naming Parameters should be formatted as snake_case by default. Hyphens or camel @@ -147,8 +173,13 @@ with the help command `nb help `. For example, if a driver module contains `../src/main/resources/mydriver-specials.md`, then a user would be able to find this help by running `nb help mydriver-specials`. +These sources of documentation can be wired into the main NoSQLBench documentation system with a set +of content descriptors. + ## Named Scenarios +Conformant driver implementations should come with one or more examples of a workload under the +activities directory path. Useful driver implementations should come with one or more examples of a workloads under the activities directory path. These examples should employ the "named scenarios" format as described in the main docs. By @@ -177,6 +208,9 @@ option in addition to the `--list-scenarios` command. ## Testing and Docs +Complete driver implementations should also come with a set of examples under the examples +directory path. + Unit testing within the NB code base is necessary in many places, but not in others. Use your judgement about when to *not* add unit testing, but default to adding it when it seems subjective. A treatise on when and how diff --git a/devdocs/sketches/opdata.puml b/devdocs/sketches/opdata.puml new file mode 100644 index 000000000..f1092f6bd --- /dev/null +++ b/devdocs/sketches/opdata.puml @@ -0,0 +1,8 @@ +@startuml +actor a as activity +actor nd as "NB Driver" +actor pd as "Native Driver" +actor r as "ReadyOp" +actor op as "op" + +@enduml \ No newline at end of file diff --git a/devdocs/sketches/transforms.dot b/devdocs/sketches/transforms.dot new file mode 100644 index 000000000..423edc445 --- /dev/null +++ b/devdocs/sketches/transforms.dot @@ -0,0 +1,44 @@ + +digraph { +// rankdir=LR; + node [shape = none] + + + cycles[label="read input"] + c_err[shape="none",label="[ERR,null]"] + c_ok[shape="none",label="[OK,cycle] (OpCycle)"] + + cycles -> c_err + cycles -> c_ok + + + bind_template[label="bind template"] + template_ok[label="[OK,template] (OpTemplate)"] + template_err[label="[ERR,template]"] + c_ok -> bind_template + bind_template -> template_err + bind_template -> template_ok + + exec_cmd[label="execute command"] + command_err[label="[ERR,template]"] + command_ok[label="[OK,result] (OpResult)"] + template_ok -> exec_cmd + exec_cmd -> command_err + exec_cmd -> command_ok + + verify_result[label="verify result"] + command_ok -> verify_result + result_invalid[label="[ERR,result]"] + result_ok[label="[OK,result]"] + verify_result -> result_invalid + verify_result -> result_ok + + + + +// Errornone +// ⁅⁆⟦⟧ ⟬⟭ ⟮⟯ ⟨⟩ ⁅⁆ +// { +// } + +} \ No newline at end of file diff --git a/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java b/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java index 965e1636d..7c3be7be7 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java @@ -55,7 +55,7 @@ public class NBWebServer implements Runnable { private String bindHost = "localhost"; private int bindPort = 12345; - private Map contextParams = new LinkedHashMap<>(); + private final Map contextParams = new LinkedHashMap<>(); public NBWebServer withContextParams(Map cp) { this.contextParams.putAll(cp); diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlAsyncAction.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlAsyncAction.java index eaa325e2d..43f16f7ef 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlAsyncAction.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlAsyncAction.java @@ -79,7 +79,7 @@ public class CqlAsyncAction extends BaseAsyncAction { @Override public void startOpCycle(TrackedOp opc) { - CqlOpData cqlop = opc.getData(); + CqlOpData cqlop = opc.getOpData(); long cycle = opc.getCycle(); // bind timer covers all statement selection and binding, skipping, transforming logic @@ -123,7 +123,7 @@ public class CqlAsyncAction extends BaseAsyncAction { public void onSuccess(StartedOp sop) { - CqlOpData cqlop = sop.getData(); + CqlOpData cqlop = sop.getOpData(); HashedCQLErrorHandler.resetThreadStatusCode(); if (cqlop.skipped) { @@ -217,7 +217,7 @@ public class CqlAsyncAction extends BaseAsyncAction { public void onFailure(StartedOp startedOp) { - CqlOpData cqlop = startedOp.getData(); + CqlOpData cqlop = startedOp.getOpData(); long serviceTime = startedOp.getCurrentServiceTimeNanos(); // Even if this is retryable, we expose error events diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/Save.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/Save.java index d8430c0c2..5517f622c 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/Save.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/rowoperators/Save.java @@ -20,7 +20,7 @@ public class Save implements RowCycleOperator { ThreadLocal> tl_objectMap = SharedState.tl_ObjectMap; - private String[] varnames; + private final String[] varnames; public Save(String... varnames) { this.varnames = varnames; diff --git a/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/errorhandling/NBCycleErrorHandler.java b/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/errorhandling/NBCycleErrorHandler.java index 0c6e594e4..0f6fb88c6 100644 --- a/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/errorhandling/NBCycleErrorHandler.java +++ b/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/errorhandling/NBCycleErrorHandler.java @@ -43,8 +43,8 @@ public class NBCycleErrorHandler implements CycleErrorHandler i @Override public void startOpCycle(TrackedOp opc) { - opc.getData().log("starting at " + System.nanoTime()); - opc.getData().setSimulatedDelayNanos(delayFunc.applyAsLong(opc.getCycle())); + opc.getOpData().log("starting at " + System.nanoTime()); + opc.getOpData().setSimulatedDelayNanos(delayFunc.applyAsLong(opc.getCycle())); StartedOp started = opc.start(); opQueue.add(started); } @@ -239,7 +239,7 @@ public class AsyncDiagAction extends BaseAsyncAction i try { opc=queue.take(); - DiagOpData op = opc.getData(); + DiagOpData op = opc.getOpData(); long now = System.nanoTime(); long simulatedCompletionTime = opc.getStartedAtNanos() + op.getSimulatedDelayNanos(); @@ -269,7 +269,7 @@ public class AsyncDiagAction extends BaseAsyncAction i logger.info("processing stride output for " + completedOps.get(0).getCycle()); long start = completedOps.get(0).getCycle(); long endPlus = completedOps.get(completedOps.size()-1).getCycle()+1; - String diagLog = completedOps.get(0).getData().getDiagLog().stream().collect(Collectors.joining("\n")); + String diagLog = completedOps.get(0).getOpData().getDiagLog().stream().collect(Collectors.joining("\n")); activity.getSequenceBlocker().awaitAndRun(start, endPlus, () -> logger.info(" => " + start + " -> " + endPlus + ": " + diagLog)); } } diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpAsyncOp.java b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpAsyncOp.java new file mode 100644 index 000000000..512f6a864 --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpAsyncOp.java @@ -0,0 +1,36 @@ +package io.nosqlbench.activitytype.cmds; + +import io.nosqlbench.activitytype.http.async.HttpAsyncAction; + +import java.net.http.HttpClient; + +public class HttpAsyncOp { + public final HttpAsyncAction action; + public final ReadyHttpOp op; + public final long cycle; + + private final HttpOp httpOp; + private final HttpClient client; + + public HttpAsyncOp(HttpAsyncAction action, ReadyHttpOp op, long cycle, HttpClient client) { + this.action = action; + this.op = op; + this.cycle = cycle; + this.client = client; + this.httpOp = op.apply(cycle); + + } + + public HttpOp getOp() { + return httpOp; + } + + public HttpClient getClient() { + return client; + } + + public HttpAsyncAction getAction() { + return action; + } + +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpOp.java b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpOp.java new file mode 100644 index 000000000..50893bda1 --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpOp.java @@ -0,0 +1,15 @@ +package io.nosqlbench.activitytype.cmds; + +import java.net.http.HttpRequest; + +public class HttpOp { + public final String ok_status; + public final String ok_body; + public final HttpRequest request; + + public HttpOp(HttpRequest request, String ok_status, String ok_body) { + this.request = request; + this.ok_status = ok_status; + this.ok_body = ok_body; + } +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequest.java b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpOp.java similarity index 77% rename from driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequest.java rename to driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpOp.java index 4d4cb1418..93111a957 100644 --- a/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequest.java +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpOp.java @@ -7,19 +7,21 @@ import io.nosqlbench.nb.api.errors.BasicError; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.LongFunction; -public class ReadyHttpRequest implements LongFunction { +public class ReadyHttpOp implements LongFunction { private final CommandTemplate propertyTemplate; // only populated if there is no value which is an actual bindings template - private final HttpRequest cachedRequest; + private final HttpOp cachedOp; - public ReadyHttpRequest(OpTemplate stmtDef) { + public ReadyHttpOp(OpTemplate stmtDef) { propertyTemplate = new CommandTemplate(stmtDef, List.of( HttpFormatParser::parseUrl, @@ -28,18 +30,18 @@ public class ReadyHttpRequest implements LongFunction { ); if (propertyTemplate.isStatic()) { - cachedRequest = apply(0); + cachedOp = apply(0); } else { - cachedRequest = null; + cachedOp = null; } } @Override - public HttpRequest apply(long value) { + public HttpOp apply(long value) { // If the request is invariant, simply return it, since it is thread-safe - if (this.cachedRequest != null) { - return this.cachedRequest; + if (this.cachedOp != null) { + return this.cachedOp; } Map cmd = propertyTemplate.getCommand(value); @@ -76,12 +78,20 @@ public class ReadyHttpRequest implements LongFunction { } } + String ok_status = cmd.remove("ok-status"); + String ok_body = cmd.remove("ok-body"); + + String timeoutStr = cmd.remove("timeout"); + if (timeoutStr!=null) { + builder.timeout(Duration.of(Long.parseLong(timeoutStr), ChronoUnit.MILLIS)); + } + if (cmd.size()>0) { throw new BasicError("Some provided request fields were not used: " + cmd.toString()); } HttpRequest request = builder.build(); - return request; + return new HttpOp(request, ok_status, ok_body); } } diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/ClientScope.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/ClientScope.java new file mode 100644 index 000000000..0f1a07f9c --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/ClientScope.java @@ -0,0 +1,6 @@ +package io.nosqlbench.activitytype.http; + +public enum ClientScope { + thread, + activity +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java index 0d6381c2a..3a306f479 100644 --- a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java @@ -1,18 +1,17 @@ package io.nosqlbench.activitytype.http; import com.codahale.metrics.Timer; -import io.nosqlbench.activitytype.cmds.ReadyHttpRequest; +import io.nosqlbench.activitytype.cmds.HttpOp; +import io.nosqlbench.activitytype.cmds.ReadyHttpOp; import io.nosqlbench.engine.api.activityapi.core.SyncAction; import io.nosqlbench.engine.api.activityapi.planning.OpSequence; import io.nosqlbench.engine.api.activityimpl.ActivityDef; -import io.nosqlbench.engine.api.templating.CommandTemplate; import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.virtdata.core.templates.StringBindings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -20,8 +19,6 @@ import java.nio.file.Path; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class HttpAction implements SyncAction { @@ -30,13 +27,14 @@ public class HttpAction implements SyncAction { private final HttpActivity httpActivity; private final int slot; - private int maxTries = 1; + private final int maxTries = 1; private boolean showstmts; - private OpSequence sequencer; + private OpSequence sequencer; private HttpClient client; - private HttpResponse.BodyHandler bodyreader = HttpResponse.BodyHandlers.ofString(); - private long timeoutMillis=30000L; + + private final HttpResponse.BodyHandler bodyreader = HttpResponse.BodyHandlers.ofString(); + private final long timeoutMillis=30000L; public HttpAction(ActivityDef activityDef, int slot, HttpActivity httpActivity) { @@ -46,9 +44,12 @@ public class HttpAction implements SyncAction { @Override public void init() { - this.sequencer = httpActivity.getOpSequence(); - this.client = HttpClient.newHttpClient(); + this.sequencer = httpActivity.getSequencer(); + this.client = initClient(httpActivity.getClientScope()); + } + private HttpClient initClient(ClientScope clientScope) { + return httpActivity.getClient().apply(Thread.currentThread()); } @Override @@ -62,7 +63,7 @@ public class HttpAction implements SyncAction { // op construction // The request to be used must be constructed from the template each time. - HttpRequest request=null; + HttpOp httpOp=null; // A specifier for what makes a response ok. If this is provided, then it is // either a list of valid http status codes, or if non-numeric, a regex for the body @@ -71,10 +72,19 @@ public class HttpAction implements SyncAction { String ok; try (Timer.Context bindTime = httpActivity.bindTimer.time()) { - ReadyHttpRequest readyHttpRequest = httpActivity.getOpSequence().get(cycleValue); - request =readyHttpRequest.apply(cycleValue); + ReadyHttpOp readHTTPOperation = httpActivity.getSequencer().get(cycleValue); + httpOp =readHTTPOperation.apply(cycleValue); } catch (Exception e) { - throw new RuntimeException("while binding request in cycle " + cycleValue + ": ",e); + throw new RuntimeException("while binding request in cycle " + cycleValue + ": " + e.getMessage(),e); + } finally { + if (httpActivity.isDiagnosticMode()) { + System.out.println("==== cycle " + cycleValue + " DIAGNOSTICS ===="); + if (httpOp!=null) { + httpActivity.console.summarizeRequest(httpOp.request,System.out,cycleValue); + } else { + System.out.println("---- REQUEST was null"); + } + } } int tries = 0; @@ -83,18 +93,49 @@ public class HttpAction implements SyncAction { CompletableFuture> responseFuture; try (Timer.Context executeTime = httpActivity.executeTimer.time()) { - responseFuture = client.sendAsync(request, this.bodyreader); + responseFuture = client.sendAsync(httpOp.request, this.bodyreader); } catch (Exception e) { throw new RuntimeException("while waiting for response in cycle " + cycleValue + ":" + e.getMessage(), e); } - HttpResponse response; - try (Timer.Context resultTime = httpActivity.resultTimer.time()) { - response = responseFuture.get(httpActivity.getTimeoutMs(), TimeUnit.MILLISECONDS); + HttpResponse response=null; + long startat = System.nanoTime(); + Exception error = null; + try { + response = responseFuture.get(httpActivity.getTimeoutMillis(), TimeUnit.MILLISECONDS); + if (httpOp.ok_status!=null) { + if (!String.valueOf(response.statusCode()).matches(httpOp.ok_status)) { + throw new InvalidStatusCodeException(cycleValue,httpOp.ok_status,response.statusCode()); + } + } + if (httpOp.ok_body!=null) { + if (!response.body().matches(httpOp.ok_body)) { + throw new InvalidResponseBodyException(cycleValue, httpOp.ok_body, response.body()); + } + } } catch (Exception e) { - throw new RuntimeException("while waiting for response in cycle " + cycleValue + ":", e); + error = new RuntimeException("while waiting for response in cycle " + cycleValue + ":" + e.getMessage(), e); + } finally { + long nanos = System.nanoTime() - startat; + httpActivity.resultTimer.update(nanos, TimeUnit.NANOSECONDS); + if (error==null) { + httpActivity.resultSuccessTimer.update(nanos, TimeUnit.NANOSECONDS); + } + if (httpActivity.isDiagnosticMode()) { + if (response!=null) { + httpActivity.console.summarizeResponse(response,System.out,cycleValue,nanos); + } else { + System.out.println("---- RESPONSE was null"); + } + System.out.println(); + } + + if (error!=null) { + // count and log exception types + } } + // if (ok == null) { // if (response.statusCode() != 200) { // throw new ResponseError("Result had status code " + @@ -205,4 +246,4 @@ public class HttpAction implements SyncAction { } -} \ No newline at end of file +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java index 779cf2a98..ea47e721e 100644 --- a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java @@ -3,32 +3,31 @@ package io.nosqlbench.activitytype.http; import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; -import io.nosqlbench.activitytype.cmds.ReadyHttpRequest; +import io.nosqlbench.activitytype.cmds.ReadyHttpOp; import io.nosqlbench.engine.api.activityapi.core.Activity; import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver; import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.engine.api.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; import io.nosqlbench.engine.api.metrics.ActivityMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.http.HttpClient; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; + public class HttpActivity extends SimpleActivity implements Activity, ActivityDefObserver { private final static Logger logger = LoggerFactory.getLogger(HttpActivity.class); private final ActivityDef activityDef; + public HttpConsoleFormats console; - public StmtsDocList getStmtsDocList() { - return stmtsDocList; - } + // Used when sclientScope == ClientScope.activity + private HttpClient activityClient; + private ClientScope clientScope = ClientScope.activity; - private StmtsDocList stmtsDocList; - - - private int stride; - private Integer maxTries; - private long timeout_ms = 30_000L; - private Boolean showstmnts; public Timer bindTimer; public Timer executeTimer; public Histogram triesHisto; @@ -37,10 +36,9 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe public Histogram skippedTokens; public Timer resultSuccessTimer; - private String[] hosts; - private int port; - - private OpSequence opSequence; + private OpSequence sequencer; + private boolean diagnosticsEnabled; + private long timeout = Long.MAX_VALUE; public HttpActivity(ActivityDef activityDef) { super(activityDef); @@ -48,59 +46,80 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe } - @Override public void initActivity() { super.initActivity(); -// stride = activityDef.getParams().getOptionalInteger("stride").orElse(1); - maxTries = activityDef.getParams().getOptionalInteger("maxTries").orElse(1); - timeout_ms = activityDef.getParams().getOptionalLong("timeout_ms").orElse(30_000L); -// showstmnts = activityDef.getParams().getOptionalBoolean("showstmnts").orElse(false); - -// hosts = activityDef.getParams().getOptionalString("host").orElse("localhost").split(","); -// port = activityDef.getParams().getOptionalInteger("port").orElse(80); - - this.opSequence = createOpSequence(ReadyHttpRequest::new); - this.setDefaultsFromOpSequence(opSequence); - bindTimer = ActivityMetrics.timer(activityDef, "bind"); executeTimer = ActivityMetrics.timer(activityDef, "execute"); resultTimer = ActivityMetrics.timer(activityDef, "result"); triesHisto = ActivityMetrics.histogram(activityDef, "tries"); rowCounter = ActivityMetrics.meter(activityDef, "rows"); skippedTokens = ActivityMetrics.histogram(activityDef, "skipped-tokens"); - resultSuccessTimer = ActivityMetrics.timer(activityDef,"result-success"); - + resultSuccessTimer = ActivityMetrics.timer(activityDef, "result-success"); + this.sequencer = createOpSequence(ReadyHttpOp::new); + setDefaultsFromOpSequence(sequencer); onActivityDefUpdate(activityDef); } @Override public synchronized void onActivityDefUpdate(ActivityDef activityDef) { super.onActivityDefUpdate(activityDef); + String[] diag = getParams().getOptionalString("diag").orElse("").split(","); + Set diags = new HashSet(Arrays.asList(diag)); + this.console = new HttpConsoleFormats(diags); + this.diagnosticsEnabled = console.isDiagnosticMode(); + this.timeout = getParams().getOptionalLong("timeout").orElse(Long.MAX_VALUE); + + getParams().getOptionalString("client_scope") + .map(ClientScope::valueOf) + .ifPresent(this::setClientScope); + } - public Integer getMaxTries() { - return maxTries; + public long getTimeoutMillis() { + return timeout; } - public Boolean getShowstmts() { - return showstmnts; + private void setClientScope(ClientScope clientScope) { + this.clientScope = clientScope; } - public String[] getHosts() { - return hosts; + public ClientScope getClientScope() { + return clientScope; } - public int getPort() { - return port; + public synchronized Function getClient() { + switch (getClientScope()) { + case thread: + return t -> newClient(); + case activity: + if (this.activityClient == null) { + this.activityClient = newClient(); + } + return t -> this.activityClient; + default: throw new RuntimeException("unable to recoginize client scope: " + getClientScope()); + } } - public OpSequence getOpSequence() { - return opSequence; + public HttpClient newClient() { + HttpClient.Builder builder = HttpClient.newBuilder(); + getParams().getOptionalString("follow_redirects") + .map(String::toUpperCase) + .map(HttpClient.Redirect::valueOf) + .map(r -> { + logger.debug("follow_redirects=>" + r); + return r; + }) + .ifPresent(builder::followRedirects); + return builder.build(); } - public long getTimeoutMs() { - return timeout_ms; + public OpSequence getSequencer() { + return sequencer; + } + + public boolean isDiagnosticMode() { + return diagnosticsEnabled; } } diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpConsoleFormats.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpConsoleFormats.java new file mode 100644 index 000000000..d59703e80 --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpConsoleFormats.java @@ -0,0 +1,93 @@ +package io.nosqlbench.activitytype.http; + +import java.io.PrintStream; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class HttpConsoleFormats { + + private final Set includes; + private final long modulo; + + public HttpConsoleFormats(Set includes) { + long mod = 1L; + Set incl = new HashSet<>(); + + for (String include : includes) { + if (include.matches("[0-9]+")) { + mod = Long.parseLong(include); + } else if (include.toLowerCase().equals("all")) { + incl.add("headers"); + incl.add("stats"); + incl.add("content"); + } { + incl.add(include); + } + } + this.includes = incl; + this.modulo = mod; + } + + public void summarizeRequest(HttpRequest request, PrintStream out, long cycle) { + if ((cycle%modulo)!=0) { + return; + } + out.println("---- REQUEST cycle=" + cycle); + out.println(" --- " + request.method() + " " + request.uri() + " " + request.version().orElse(HttpClient.Version.HTTP_2)); + + if (includes.contains("headers")) { + out.println(" -- headers:"); + summariseHeaders(request.headers(),out); + } + + out.println(" -- body length:" + request.bodyPublisher().get().contentLength()); + } + + public void summarizeResponse(HttpResponse response, PrintStream out, long cycle, long nanos) { + if ((cycle%modulo)!=0) { + return; + } + out.println("---- RESPONSE for cycle=" + cycle + " status=" + response.statusCode() + " took=" + (nanos/1_000_000) + "ms"); + + if (includes.contains("stats")) { + int redirects=0; + Optional> walkResponses = response.previousResponse(); + while (walkResponses.isPresent()) { + walkResponses=walkResponses.get().previousResponse(); + redirects++; + } + System.out.println(" redirects = " + redirects); + } + + summariseHeaders(response.headers(),out); + + if (this.includes.contains("content")) { + System.out.println(" -- body:"); + System.out.println(response.body()); + } + } + + private static void summariseHeaders(HttpHeaders headers, PrintStream out) { + out.println(" --- headers:"); + headers.map().forEach((k,v) -> { + out.print(" --- " + k + ":"); + if (v.size()>1) { + out.println(); + v.forEach( h -> { + out.println(" - " + h); + }); + } else { + out.println(" " + v.get(0)); + } + }); + } + + public boolean isDiagnosticMode() { + return this.includes.size()>0; + } +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/InvalidResponseBodyException.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/InvalidResponseBodyException.java new file mode 100644 index 000000000..8d1146773 --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/InvalidResponseBodyException.java @@ -0,0 +1,13 @@ +package io.nosqlbench.activitytype.http; + +public class InvalidResponseBodyException extends RuntimeException { + private final long cycleValue; + private final String ok_body; + private final String body; + + public InvalidResponseBodyException(long cycleValue, String ok_body, String body) { + this.cycleValue = cycleValue; + this.ok_body = ok_body; + this.body = body; + } +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/InvalidStatusCodeException.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/InvalidStatusCodeException.java new file mode 100644 index 000000000..471250b50 --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/InvalidStatusCodeException.java @@ -0,0 +1,13 @@ +package io.nosqlbench.activitytype.http; + +public class InvalidStatusCodeException extends RuntimeException { + private final long cycleValue; + private final String ok_status; + private final int statusCode; + + public InvalidStatusCodeException(long cycleValue, String ok_status, int statusCode) { + this.cycleValue = cycleValue; + this.ok_status = ok_status; + this.statusCode = statusCode; + } +} diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/async/HttpAsyncAction.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/async/HttpAsyncAction.java new file mode 100644 index 000000000..676e335e6 --- /dev/null +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/async/HttpAsyncAction.java @@ -0,0 +1,61 @@ +package io.nosqlbench.activitytype.http.async; + +import io.nosqlbench.activitytype.cmds.HttpAsyncOp; +import io.nosqlbench.activitytype.cmds.HttpOp; +import io.nosqlbench.activitytype.cmds.ReadyHttpOp; +import io.nosqlbench.activitytype.http.HttpActivity; +import io.nosqlbench.engine.api.activityapi.core.BaseAsyncAction; +import io.nosqlbench.engine.api.activityapi.core.ops.fluent.opfacets.TrackedOp; +import io.nosqlbench.engine.api.activityapi.planning.OpSequence; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.concurrent.CompletableFuture; +import java.util.function.LongFunction; + +public class HttpAsyncAction extends BaseAsyncAction { + + private final static Logger logger = LoggerFactory.getLogger(HttpAsyncAction.class); + + private OpSequence sequencer; + private HttpClient client; + + private CompletableFuture> future; + + + public HttpAsyncAction(HttpActivity httpActivity, int slot) { + super(httpActivity,slot); + } + + @Override + public void startOpCycle(TrackedOp opc) { + HttpAsyncOp opdata = opc.getOpData(); + HttpOp op = opdata.getOp(); + + opc.start(); + future = opdata.getAction().client.sendAsync(op.request, HttpResponse.BodyHandlers.discarding()); + } + + public void init() { + this.sequencer = activity.getSequencer(); + this.client = activity.getClient().apply(Thread.currentThread()); + } + + @Override + public LongFunction getOpInitFunction() { + return l -> { + ReadyHttpOp readyHttpOp = sequencer.get(l); + return new HttpAsyncOp(this,readyHttpOp,l,client); + }; + } + + +// @Override +// public boolean enqueue(TrackedOp opc) { +// HttpAsyncOp opData = opc.getOpData(); +// opData.op. +// return false; +// } +} diff --git a/driver-http/src/main/resources/http.md b/driver-http/src/main/resources/http.md index 9b0e4e0ee..b00096fd2 100644 --- a/driver-http/src/main/resources/http.md +++ b/driver-http/src/main/resources/http.md @@ -41,7 +41,7 @@ bindings: You can even make a detailed request with custom headers and result verification conditions: ```yaml -# Require that the result be status code 200-299 match regex "OK, account id is .*" in the body +# Require that the result be status code 200-299 match regex "OK, account id is .*" in the body statements: - method: GET uri: https://google.com/ @@ -62,7 +62,7 @@ statements: GET https://google.com/ HTTP/1.1 Content-Type: application/json ok-status: 2[0-9][0-9] - ok-body: ^(OK, account id is.*)$ + ok-body: ^(OK, account id is.*)$ ``` Of course, in the above form, the response validators are still separate parameters. @@ -78,8 +78,8 @@ statements: {method} {scheme}://{host}:{port}/{path}?{query} {version} Content-Type: {content_type} Token: {mybearertoken} - - {body} + + {body} ``` The above example is in the inline request form. It is parsed and interpreted internally as if you @@ -157,3 +157,25 @@ the defaults for the current HttpClient library that is bundled within the JVM. Presently, this driver only does basic request-response style requests. Thus, adding headers which take TCP socket control away from the HttpClient will likely yield inconsistent (or undefined) results. Support may be added for long-lived connections in a future release. + +## HTTP Activity Parameters + +- **client_scope** - default: activity - One of activity, or thread. This controls how many + clients instances you use with an HTTP activity. By default, all threads will use the same + client instance. +- **follow_redirects** - default: normal - One of never, always, or normal. Normal redirects + are those which do not redirect from HTTPS to HTTP. +- **diagnostics** - default: none - + This setting is a selector for what level of verbosity you will get on console. If you set + this to true, you'll get every request and response logged to console. This is only for + verifying that a test is configured and to spot check services before running higher scale + tests. + If you want finer control over how much information diagnostics provides, you can specify + a comma separated list of the below. + - all - Includes all of the below categories + - stats - Counts of redirects, headers, body length, etc + - headers - include header details + - content - include + - a number, like 3000 - causes the diagnostics to be reported only on this cycle modulo +- **timeout** - default: tbd - + Sets the diff --git a/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequestTest.java b/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpOpTest.java similarity index 84% rename from driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequestTest.java rename to driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpOpTest.java index cbea4ee7c..1c85feb1c 100644 --- a/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequestTest.java +++ b/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpOpTest.java @@ -10,7 +10,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -public class ReadyHttpRequestTest { +public class ReadyHttpOpTest { @Test public void testOnelineSpec() { @@ -19,8 +19,8 @@ public class ReadyHttpRequestTest { " - s1: method=get uri=http://localhost/\n"); OpTemplate stmtDef = docs.getStmts().get(0); - ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef); - HttpRequest staticReq = readyReq.apply(3); + ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef); + HttpOp staticReq = readyReq.apply(3); } @Test @@ -30,8 +30,8 @@ public class ReadyHttpRequestTest { " - s1: get http://localhost/"); OpTemplate stmtDef = docs.getStmts().get(0); - ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef); - HttpRequest staticReq = readyReq.apply(3); + ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef); + HttpOp staticReq = readyReq.apply(3); } @Test @@ -41,8 +41,8 @@ public class ReadyHttpRequestTest { " - s1: get http://localhost/ HTTP/1.1"); OpTemplate stmtDef = docs.getStmts().get(0); - ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef); - HttpRequest staticReq = readyReq.apply(3); + ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef); + HttpOp staticReq = readyReq.apply(3); } @Test @@ -55,8 +55,8 @@ public class ReadyHttpRequestTest { ""); OpTemplate stmtDef = docs.getStmts().get(0); - ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef); - HttpRequest staticReq = readyReq.apply(3); + ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef); + HttpOp staticReq = readyReq.apply(3); } @Test @@ -69,8 +69,8 @@ public class ReadyHttpRequestTest { " body1"); OpTemplate stmtDef = docs.getStmts().get(0); - ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef); - HttpRequest staticReq = readyReq.apply(3); + ReadyHttpOp readyReq = new ReadyHttpOp(stmtDef); + HttpOp staticReq = readyReq.apply(3); } @Test diff --git a/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActionDispenser.java b/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActionDispenser.java index d0a2c6ac5..71d4f2678 100644 --- a/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActionDispenser.java +++ b/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/JMXActionDispenser.java @@ -4,7 +4,7 @@ import io.nosqlbench.engine.api.activityapi.core.Action; import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; public class JMXActionDispenser implements ActionDispenser { - private JMXActivity activity; + private final JMXActivity activity; public JMXActionDispenser(JMXActivity activity) { this.activity = activity; diff --git a/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/formats/MBeanInfoConsoleFormat.java b/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/formats/MBeanInfoConsoleFormat.java index c38c92cc2..72c29b1f3 100644 --- a/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/formats/MBeanInfoConsoleFormat.java +++ b/driver-jmx/src/main/java/io/nosqlbench/driver/jmx/formats/MBeanInfoConsoleFormat.java @@ -5,7 +5,7 @@ import java.util.Map; public class MBeanInfoConsoleFormat { - private static Map MbeanOpImpacts = Map.of( + private static final Map MbeanOpImpacts = Map.of( MBeanOperationInfo.ACTION,"ACTION", MBeanOperationInfo.ACTION_INFO,"ACTION_INFO", MBeanOperationInfo.UNKNOWN,"UNKNOWN", diff --git a/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/AsyncStdoutAction.java b/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/AsyncStdoutAction.java index 67f2f91b6..89c9363c2 100644 --- a/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/AsyncStdoutAction.java +++ b/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/AsyncStdoutAction.java @@ -46,7 +46,7 @@ public class AsyncStdoutAction extends BaseAsyncAction started = opc.start(); int result=0; try (Timer.Context executeTime = activity.executeTimer.time()) { - activity.write(opc.getData().statement); + activity.write(opc.getOpData().statement); } catch (Exception e) { result=1; started.fail(result); diff --git a/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java b/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java index 6957fa4bb..82aa41aed 100644 --- a/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java +++ b/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java @@ -56,7 +56,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve public Timer resultTimer; public Histogram triesHisto; private Writer pw; - private String fileName; + private final String fileName; private ExceptionMeterMetrics exceptionMeterMetrics; private int retry_delay = 0; private int retries; diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/OpImpl.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/OpImpl.java index f30246302..e4011c75c 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/OpImpl.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/OpImpl.java @@ -100,7 +100,7 @@ public class OpImpl implements OpFacets { } @Override - public D getData() { + public D getOpData() { return data; } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/Payload.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/Payload.java index 5e193262f..036e8c807 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/Payload.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ops/fluent/opfacets/Payload.java @@ -23,6 +23,6 @@ package io.nosqlbench.engine.api.activityapi.core.ops.fluent.opfacets; * @param The type of delegate needed for the implementing protocol */ public interface Payload { - D getData(); + D getOpData(); void setData(D data); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/sysperf/SysPerf.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/sysperf/SysPerf.java index b77cf7f83..e5f64e208 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/sysperf/SysPerf.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/sysperf/SysPerf.java @@ -24,6 +24,7 @@ import org.yaml.snakeyaml.Yaml; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileTime; @@ -32,9 +33,9 @@ import java.util.Optional; public class SysPerf { public final static Logger logger = LoggerFactory.getLogger(SysPerf.class); - private static final Charset CHARSET = Charset.forName("UTF8"); + private static final Charset CHARSET = StandardCharsets.UTF_8; private static SysPerfData cachedData; - private static long currentImplVersion = 1L; + private static final long currentImplVersion = 1L; private static SysPerf instance; private SysPerf() { @@ -112,7 +113,7 @@ public class SysPerf { String perfdata = new String(bytes, CHARSET); Yaml yaml = new Yaml(); - SysPerfData perfinfo = (SysPerfData) yaml.load(perfdata); + SysPerfData perfinfo = yaml.load(perfdata); cachedData = perfinfo; logger.info("Loaded previously cached system timing data from " + cache.getCanonicalPath()); return cachedData; diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java index 1b847cb72..dcee603fc 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java @@ -33,7 +33,7 @@ import java.util.Map; public class RawStmtsDoc extends StatementsOwner { private RawScenarios scenarios = new RawScenarios(); - private List blocks = new ArrayList<>(); + private final List blocks = new ArrayList<>(); // no-args ctor is required public RawStmtsDoc() { diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java index 641d70b4e..dc08c6cf6 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java @@ -21,7 +21,7 @@ import java.util.List; public class RawStmtsDocList { - private List rawStmtsDocList; + private final List rawStmtsDocList; public RawStmtsDocList(List rawStmtsDocList) { this.rawStmtsDocList = rawStmtsDocList; diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java index da9c14464..8a542f7f7 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/ParameterMap.java @@ -166,9 +166,9 @@ public class ParameterMap extends ConcurrentHashMap implements Bi markMutation(); } - private static Pattern encodedParamsSquote = Pattern.compile("(?\\w+?)='(?[^']+?);"); - private static Pattern encodedParamsDquote = Pattern.compile("(?\\w+?)=\"(?[^\"]+?);"); - private static Pattern encodedParamsPattern = Pattern.compile("(?\\w+?)=(?.+?);"); + private static final Pattern encodedParamsSquote = Pattern.compile("(?\\w+?)='(?[^']+?);"); + private static final Pattern encodedParamsDquote = Pattern.compile("(?\\w+?)=\"(?[^\"]+?);"); + private static final Pattern encodedParamsPattern = Pattern.compile("(?\\w+?)=(?.+?);"); @Override public Object put(String name, Object value) { @@ -345,7 +345,7 @@ public class ParameterMap extends ConcurrentHashMap implements Bi // return parameterMap; // } - public static interface Listener { + public interface Listener { void handleParameterMapUpdate(ParameterMap parameterMap); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/clireader/dsl/CLIFacets.java b/engine-api/src/main/java/io/nosqlbench/engine/api/clireader/dsl/CLIFacets.java index 517dd0f3b..bca3f38b4 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/clireader/dsl/CLIFacets.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/clireader/dsl/CLIFacets.java @@ -2,22 +2,22 @@ package io.nosqlbench.engine.api.clireader.dsl; public interface CLIFacets { - public static interface WantsAnyOption + interface WantsAnyOption extends WantsGlobalOption { } - public static interface WantsParameterizedCommand { - public WantsAnyOption namedParams(); + interface WantsParameterizedCommand { + WantsAnyOption namedParams(); } - public static interface WantsGlobalOption { - public WantsOptionType global(String optionName); + interface WantsGlobalOption { + WantsOptionType global(String optionName); } - public static interface WantsOptionType { - public WantsAnyOption toggle(); - public WantsAnyOption string(); - public WantsAnyOption number(); + interface WantsOptionType { + WantsAnyOption toggle(); + WantsAnyOption string(); + WantsAnyOption number(); } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/NashornEvaluator.java b/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/NashornEvaluator.java index 68d34c00b..7622eab28 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/NashornEvaluator.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/scripting/NashornEvaluator.java @@ -34,7 +34,7 @@ public class NashornEvaluator implements Evaluator { private final ScriptEngine scriptEngine; private final SimpleBindings bindings = new SimpleBindings(); private String script = ""; - private Class resultType; + private final Class resultType; private CompiledScript compiled; /** diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/util/TagFilter.java b/engine-api/src/main/java/io/nosqlbench/engine/api/util/TagFilter.java index 63a21b71d..418167fb1 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/util/TagFilter.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/util/TagFilter.java @@ -92,7 +92,7 @@ import java.util.stream.Collectors; */ public class TagFilter { public static TagFilter MATCH_ALL = new TagFilter(""); - private Map filter = new LinkedHashMap<>(); + private final Map filter = new LinkedHashMap<>(); private Conjugate conjugate = Conjugate.all; private final static Pattern conjugateForm = Pattern.compile("^(?\\w+)\\((?.+)\\)$",Pattern.DOTALL|Pattern.MULTILINE); @@ -217,8 +217,8 @@ public class TagFilter { } public static class Result { - private boolean matched; - private List matchLog; + private final boolean matched; + private final List matchLog; public Result(boolean matched, List log) { this.matched = matched; diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java index 1c1455851..0b5482971 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java @@ -157,7 +157,7 @@ public class ScenarioLogger { } } - private static Comparator fileTimeComparator = new Comparator() { + private static final Comparator fileTimeComparator = new Comparator() { @Override public int compare(File o1, File o2) { return Long.compare(o1.lastModified(),o2.lastModified()); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java index 8d2545bf8..3f2b58065 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java @@ -32,7 +32,7 @@ public class ScenarioResult { private final static Logger logger = LoggerFactory.getLogger(ScenarioResult.class); private Exception exception; - private String iolog; + private final String iolog; public ScenarioResult(String iolog) { this.iolog = iolog; diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java b/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java index a3be7b67f..eae775033 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScriptParams.java @@ -128,7 +128,7 @@ public class ScriptParams extends HashMap implements ProxyObject @Override public String toString() { - return gson.toJson(this, Map.class).toString(); + return gson.toJson(this, Map.class); } private static String valueOf(Object o) { @@ -172,10 +172,7 @@ public class ScriptParams extends HashMap implements ProxyObject if (super.containsKey(key)) { return true; } - if (key.equals("withOverrides") || key.equals("withDefaults")) { - return true; - } - return false; + return key.equals("withOverrides") || key.equals("withDefaults"); } @Override diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspacesView.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspacesView.java index 9713e60be..bbe12389a 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspacesView.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspacesView.java @@ -9,7 +9,7 @@ import java.util.List; public class WorkspacesView { - private Path workspacesRoot; + private final Path workspacesRoot; public WorkspacesView(Path workspacesRoot) { this.workspacesRoot = workspacesRoot; } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java index ae177c49a..4c5eb4b16 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java @@ -97,7 +97,7 @@ public class PathFinder { throw new RuntimeException(e); } } - logger.debug("Found path in classpath: " + candidatePath + ": " + candidatePath.toString()); + logger.debug("Found path in classpath: " + candidatePath + ": " + candidatePath); return Optional.of(Path.of(uri)); } return Optional.empty(); diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java index 8fdeab12c..5d665b318 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java @@ -9,7 +9,7 @@ import java.util.LinkedList; import java.util.List; public class CompositeMarkdownInfo implements MarkdownInfo { - private List elements = new LinkedList<>(); + private final List elements = new LinkedList<>(); private boolean isSorted=false; @Override diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java index 9f0d903ee..824900e9e 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java @@ -204,7 +204,7 @@ public class MarkdownDocs { private final int from; private final int to; - private T edgeProps; + private final T edgeProps; public Edge(int from, int to, Supplier forT) { this.from = from; diff --git a/nb/src/main/resources/scripts/auto/findmax.js b/nb/src/main/resources/scripts/auto/findmax.js index fbf872fc7..72f9ce2d7 100644 --- a/nb/src/main/resources/scripts/auto/findmax.js +++ b/nb/src/main/resources/scripts/auto/findmax.js @@ -316,7 +316,7 @@ function find_max() { accepted_count++; highest_acceptable_iteration = highest_acceptable_of(results, highest_acceptable_iteration, iteration); printf(" ---> accepting iteration %d\n",iteration); - continue; + } else { rejected_count++; printf(" !!! rejecting iteration %d\n",iteration); diff --git a/nb/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java b/nb/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java index 07483fcd9..47743eb8d 100644 --- a/nb/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java +++ b/nb/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java @@ -48,7 +48,7 @@ public class NBCliIntegrationTests { assertThat(result.exitStatus).isEqualTo(0); } // disabled till after release - // This is not disabled on testbranch, only on master + // This is not disabled on testbranch, only on main // @Test // public void dockerMetrics() { // ProcessInvoker invoker = new ProcessInvoker(); diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_uuid/ToUUID.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_uuid/ToUUID.java index 8fc6c8f90..8bc22d78f 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_uuid/ToUUID.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_uuid/ToUUID.java @@ -26,7 +26,7 @@ public class ToUUID implements LongFunction { } public ToUUID(long msbs) { - this.msbs = (msbs & 0xFFFFFFFFFFFF0FFFL) | 0x0000000000004000L;; + this.msbs = (msbs & 0xFFFFFFFFFFFF0FFFL) | 0x0000000000004000L; } @Override diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java index 9d430ff2b..350f7d7ce 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java @@ -24,7 +24,7 @@ import java.util.stream.Collectors; public class AutoDocsWebService implements WebServiceObject { private final static Logger logger = LogManager.getLogger(AutoDocsWebService.class); - private List _docs = VirtDataDocs.getAllDocs(); + private final List _docs = VirtDataDocs.getAllDocs(); @GET @Produces(MediaType.APPLICATION_JSON)