minor bugfixes

driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java

@@ -7,7 +7,14 @@ import java.util.Map;
 
 public class HttpFormatParser {
 
-    public static Map<String,String> parse(String command) {
+    public static Map<String,String> parseUrl(String uri) {
+        if (uri.matches("http.+")) {
+            return Map.of("uri",uri);
+        }
+        return null;
+    }
+
+    public static Map<String,String> parseInline(String command) {
         if (command==null) {
             return null;
         }

driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequest.java

@@ -7,6 +7,7 @@ import io.nosqlbench.nb.api.errors.BasicError;
 import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.LongFunction;
@@ -19,8 +20,12 @@ public class ReadyHttpRequest implements LongFunction<HttpRequest> {
     private final HttpRequest cachedRequest;
 
     public ReadyHttpRequest(OpTemplate stmtDef) {
-        propertyTemplate = new CommandTemplate(stmtDef, HttpFormatParser::parse);
-        Set<String> namedProperties = propertyTemplate.getPropertyNames();
+        propertyTemplate = new CommandTemplate(stmtDef,
+                List.of(
+                        HttpFormatParser::parseUrl,
+                        HttpFormatParser::parseInline
+                )
+        );
 
         if (propertyTemplate.isStatic()) {
             cachedRequest = apply(0);
@@ -50,7 +55,10 @@ public class ReadyHttpRequest implements LongFunction<HttpRequest> {
         builder.method(method, bodyPublisher);
 
         if (cmd.containsKey("version")) {
-            HttpClient.Version version = HttpClient.Version.valueOf(cmd.remove("version"));
+            String versionName = cmd.remove("version")
+                    .replaceAll("/1.1", "_1_1")
+                    .replaceAll("/2.0", "_2");
+            HttpClient.Version version = HttpClient.Version.valueOf(versionName);
             builder.version(version);
         }
 
@@ -59,14 +67,18 @@ public class ReadyHttpRequest implements LongFunction<HttpRequest> {
             builder.uri(uri);
         }
 
-        for (String header : cmd.keySet()) {
+        Set<String> headers = cmd.keySet();
+        for (String header : headers) {
             if (header.charAt(0) >= 'A' && header.charAt(0) <= 'Z') {
-                builder.header(header, cmd.get(header));
+                builder.header(header, cmd.remove(header));
             } else {
                 throw new BasicError("HTTP request parameter '" + header + "' was not recognized as a basic request parameter, and it is not capitalized to indicate that it is a header.");
             }
         }
 
+        if (cmd.size()>0) {
+            throw new BasicError("Some provided request fields were not used: " + cmd.toString());
+        }
 
         HttpRequest request = builder.build();
         return request;

driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java

@@ -36,7 +36,7 @@ public class HttpAction implements SyncAction {
     private OpSequence<ReadyHttpRequest> sequencer;
     private HttpClient client;
     private HttpResponse.BodyHandler<String> bodyreader = HttpResponse.BodyHandlers.ofString();
-    private long timeoutMillis;
+    private long timeoutMillis=30000L;
 
 
     public HttpAction(ActivityDef activityDef, int slot, HttpActivity httpActivity) {
@@ -90,7 +90,7 @@ public class HttpAction implements SyncAction {
 
             HttpResponse<String> response;
             try (Timer.Context resultTime = httpActivity.resultTimer.time()) {
-                response = responseFuture.get(timeoutMillis, TimeUnit.MILLISECONDS);
+                response = responseFuture.get(httpActivity.getTimeoutMs(), TimeUnit.MILLISECONDS);
             } catch (Exception e) {
                 throw new RuntimeException("while waiting for response in cycle " + cycleValue + ":" + e.getMessage(), e);
             }

driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java

@@ -41,6 +41,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
 
     private int stride;
     private Integer maxTries;
+    private long timeout_ms = 30_000L;
     private Boolean showstmnts;
     public Timer bindTimer;
     public Timer executeTimer;
@@ -68,6 +69,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
 
 //        stride = activityDef.getParams().getOptionalInteger("stride").orElse(1);
         maxTries = activityDef.getParams().getOptionalInteger("maxTries").orElse(1);
+        timeout_ms = activityDef.getParams().getOptionalLong("timeout_ms").orElse(30_000L);
 //        showstmnts = activityDef.getParams().getOptionalBoolean("showstmnts").orElse(false);
 
 //        hosts = activityDef.getParams().getOptionalString("host").orElse("localhost").split(",");
@@ -111,4 +113,8 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
     public OpSequence<ReadyHttpRequest> getOpSequence() {
         return opSequence;
     }
+
+    public long getTimeoutMs() {
+        return timeout_ms;
+    }
 }

driver-http/src/main/resources/http.md

@@ -3,52 +3,127 @@
 This driver allows you to make http requests using the native HTTP client that is bundled with the
 JVM. It supports free-form construction of requests.
 
+You specify what a request looks like by providing a set of request parameters. They can be in
+either literal (static) form with no dynamic data binding, or they can each be in a string template
+form that draws from data bindings. Each cycle, a request is assembled from these parameters and
+executed.
+
 ## Example Statements
 
-You can use an _inline request template_ form below to represent a request as it would be submitted
-according to the HTTP protocol. This isn't actually the content that is submitted, but it is
-recognized as a valid way to express the request parameters in a familiar and condensed form:
+The simplest possible statement form looks like this:
 
 ```yaml
-statements:
- - s1: |
-    POST http://{host}:{port}/{path}?{query} HTTP/1.1
-    Content-Type: application/json
-    token: mybearertokenfoobarbazomgwtfbbq
-    
-    {body} 
+statement: http://google.com/
 ```
 
-You can also provide the building blocks of a request in named fields:
+Or, you can have a list:
 
 ```yaml
+# A list of statements
+statements:
+ - http://google.com/
+ - http://amazon.com/
+```
+
+Or you can template the values used in the URI, and even add ratios:
+
+```yaml
+# A list of named statements with variable fields and specific ratios:
+statements:
+ - s1: http://google.com/search?query={query}
+   ratio: 3
+ - s2: https://www.amazon.com/s?k={query}
+   ratio: 2
+bindings:
+ query: WeightedStrings('function generator;backup generator;static generator');
+```
+
+You can even make a detailed request with custom headers and result verification conditions:
+
+```yaml
+# Require that the result be status code 200-299 match regex "OK, account id is .*" in the body 
+statements:
   - method: GET
+    uri: https://google.com/
     version: HTTP/1.1
     "Content-Type": "application/json"
-    body: {body}
-    path: {path}
     ok-status: 2[0-9][0-9]
     ok-body: ^(OK, account id is .*)$
 ```
 
-As you may guess from the above example, some reserved words are recognized as standard request
-parameters. They are explained here in more detail:
+For those familiar with what an HTTP request looks like on the wire, the format below may be
+familiar. This isn't actually the content that is submitted, but it is recognized as a valid way to
+express the request parameters in a familiar and condensed form. A custom config parser makes this
+form available fo rhose who want to emulate a well-known pattern:
 
+```yaml
+statements:
+  - s1: |
+     GET https://google.com/ HTTP/1.1
+     Content-Type: application/json
+    ok-status: 2[0-9][0-9]
+    ok-body: ^(OK, account id is.*)$ 
+
+```
+
+Of course, in the above form, the response validators are still separate parameters.
+
+## Bindings
+
+All request fields can be made dynamic with binding functions. To make a request that has all
+dynamic fields, you can do something like this:
+
+```yaml
+statements:
+  - s1: |
+     {method} {scheme}://{host}:{port}/{path}?{query} {version}
+     Content-Type: {content_type}
+     Token: {mybearertoken}
+     
+     {body} 
+```
+
+The above example is in the inline request form. It is parsed and interpreted internally as if you
+had configured your op template like this:
+
+```yaml
+statements:
+  - method: {method}
+    uri: {scheme}://{host}:{port}/{path}?{query}
+    version: {version}
+    "Content-Type": {content_type}
+    "Token": {mybearertoken}
+    body: {body}
+```
+
+The above two examples are semantically identical, only the format is different. Notice that the
+expansion of the URI is still captured in a field called uri, with all of the dynamic pieces
+stitched together in the value. You can't use arbitrary request fields. Every request field must
+from (method, uri, version, body, ok-status, ok-body) or otherwise be capitalized to signify an HTTP
+header.
+
+The HTTP RFCs do not require headers to be capitalized, but they are capitalized ubiquitously in
+practice, so we follow that convention here for clarity. Headers are in-fact case-insensitive, so
+any issues created by this indicate a non-conformant server/application implementation.
+
+For URIs which are fully static (There are no dynamic fields, request generation will be much
+faster, since the request is fully built and cached at startup.
+
+
+## Request Fields
+
+At a minimum, a **URI** must be provided. These are enough to build a request with.
+All other request fields are optional and have reasonable defaults:
+
+- **uri** - This is the URI that you might put into the URL bar of your browser. There is no
+  default. Example: `https://en.wikipedia.org/wiki/Leonhard_Euler`
 - **method** - An optional request method. If not provided, "GET" is assumed. Any method name will
   work here, even custom ones that are specific to a given target system. No validation is done for
   standard method names, as there is no way to know what method names may be valid.
-- **host** - The name of the host which should go into the URI. This can also be an ip address if
-  you do not need support for virtual hosts. If there are multiple hosts provided to the activity,
-  then this value is selected in round-robin style. **default: localhost**
-- **port** - The post to connect to. If it is provided, then it is added to the URI, even if it is
-  the default for the scheme (80 for http, or 443 for https)
-- **path** - The path component of the URI.
-- **query** - A query string. If this is provided, it is appended to the path in the URI with a
-  leading question mark.
 - **version** - The HTTP version to use. If this value is not provided, the default version for the
-  Java HttpClient is used. If it is provided, it must be one of 'HTTP_1_1', or 'HTTP_2'.
+  Java HttpClient is used. If it is provided, it must be one of 'HTTP/1.1' or 'HTTP/2.0'.
 - **body** - The content of the request body, for methods which support it.
-- **ok-status** - An optional set of rules to use to verify that a response is valid. This is a
+- **ok-status** - An optional set of rules to verify that a response is valid. This is a
   simple comma or space separated list of integer status codes or a pattern which is used as a regex
   against the string form of a status code. If any characters other than digits spaces and commas
   are found in this value, then it is taken as a regex. If this is not provided, then any status
@@ -64,11 +139,12 @@ error is thrown.
 ## Error Handling & Retries
 
 Presently, no determination is made about whether or not an errored response *should* be retryable.
-Contextual error handling may be added in a future version.
+More contextual error handling may be added in a future version.
 
 ## SSL Support
 
-SSL Support will be added before this driver is considered ready for general use.
+SSL should work for any basic client request that doesn't need custom SSL configuration. If needed,
+more configurable SSL support will be added.
 
 ## Client Behavior
 

driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequestTest.java

@@ -2,7 +2,6 @@ package io.nosqlbench.activitytype.cmds;
 
 import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
-import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import org.junit.Test;
 
@@ -98,14 +97,14 @@ public class ReadyHttpRequestTest {
                 " body: StaticString('test')\n");
         OpTemplate stmtDef = docs.getStmts().get(0);
 
-        Map<String, String> parse = HttpFormatParser.parse(stmtDef.getStmt());
+        Map<String, String> parse = HttpFormatParser.parseInline(stmtDef.getStmt());
         assertThat(parse).containsAllEntriesOf(
                 Map.of(
                         "method", "{method}",
                         "uri", "{scheme}://{host}/{path}?{query}",
                         "version", "{version}",
                         "Header1","{header1val}",
-                        "body","{body}\n"
+                        "body","{body}"
                 )
         );
 

driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java

@@ -160,7 +160,7 @@ public class WebDriverActivity extends SimpleActivity {
             .orElse(SequencerType.bucket);
         SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
         commands.forEach((name,cmd) -> {
-            CommandTemplate commandTemplate = new CommandTemplate(name, cmd, Map.of(), Map.of());
+            CommandTemplate commandTemplate = new CommandTemplate(name, cmd, Map.of(), Map.of(),List.of());
             planner.addOp(commandTemplate,(c) -> 1L);
         });
         OpSequence<CommandTemplate> sequence = planner.resolve();

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/CoreServices.java

@@ -66,7 +66,7 @@ public class CoreServices {
 //    }
 //
     public static <A extends Activity> InputDispenser getInputDispenser(A activity) {
-        String inputTypeName = new SimpleConfig(activity, "input").getString("type").orElse("targetrate");
+        String inputTypeName = new SimpleConfig(activity, "input").getString("type").orElse("atomicseq");
         InputType inputType = InputType.FINDER.getOrThrow(inputTypeName);
         InputDispenser dispenser = inputType.getInputDispenser(activity);
         Optional<Predicate<ResultReadable>> inputFilterDispenser = getInputFilter(activity);

engine-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java

@@ -12,8 +12,6 @@ import org.slf4j.LoggerFactory;
 
 import java.util.*;
 import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 /**
  * Use the {@link StmtDef} template form as a property template for parameterized commands. This is a general purpose
@@ -35,11 +33,11 @@ public class CommandTemplate {
     private final Map<String, StringBindings> dynamics = new HashMap<>();
 
     public CommandTemplate(OpTemplate stmt) {
-        this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), null);
+        this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), List.of());
     }
 
-    public CommandTemplate(OpTemplate stmt, Function<String, Map<String, String>> parser) {
-        this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), parser);
+    public CommandTemplate(OpTemplate stmt, List<Function<String, Map<String, String>>> parsers) {
+        this(stmt.getName(), stmt.getStmt(), stmt.getParamsAsValueType(String.class), stmt.getBindings(), parsers);
     }
 
     /**
@@ -50,7 +48,7 @@ public class CommandTemplate {
      * @param params   A set of named parameters and values in name:value form.
      * @param bindings A set of named bindings in name:recipe form.
      */
-    public CommandTemplate(String name, String oneline, Map<String, String> params, Map<String, String> bindings, Function<String, Map<String, String>> optionalParser) {
+    public CommandTemplate(String name, String oneline, Map<String, String> params, Map<String, String> bindings, List<Function<String, Map<String, String>>> optionalParsers) {
 
         this.name = name;
 
@@ -61,7 +59,7 @@ public class CommandTemplate {
         // The first parser to match and return a map will be the last one tried.
         // If none of the suppliemental parsers work, the default params parser is used
         if (oneline != null) {
-            List<Function<String,Map<String,String>>> parserlist = new ArrayList<>(List.of(optionalParser));
+            List<Function<String,Map<String,String>>> parserlist = new ArrayList<>(optionalParsers);
             parserlist.add(s -> ParamsParser.parse(s,false));
             for (Function<String, Map<String, String>> parser : parserlist) {
                 Map<String, String> parsed = parser.apply(oneline);