workin progress for http_updates

driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpRequestSetter.java

@@ -0,0 +1,30 @@
+package io.nosqlbench.activitytype.cmds;
+
+import io.nosqlbench.engine.api.templating.EnumSetter;
+
+import java.net.http.HttpRequest;
+
+public class HttpRequestSetter implements EnumSetter<ReadyHttpRequest.FieldType,HttpRequest.Builder> {
+
+    @Override
+    public HttpRequest.Builder setField(
+            HttpRequest.Builder target,
+            ReadyHttpRequest.FieldType field,
+            Object... value) {
+        switch (field) {
+            case method:
+                return target.method((String)value[0], (HttpRequest.BodyPublisher) value[1]);
+            case host:
+            case path:
+            case query:
+            case header:
+            case version:
+                return target;
+            default:
+                throw new RuntimeException("field type was not set correctly:" + field);
+        }
+
+    }
+
+
+}

driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequest.java

@@ -0,0 +1,99 @@
+package io.nosqlbench.activitytype.cmds;
+
+import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
+import io.nosqlbench.engine.api.activityimpl.motor.ParamsParser;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
+import io.nosqlbench.nb.api.errors.BasicError;
+import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
+import io.nosqlbench.virtdata.core.templates.StringBindings;
+import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
+
+import java.net.http.HttpRequest;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.LongFunction;
+import java.util.stream.Collectors;
+
+public class ReadyHttpRequest implements LongFunction<HttpRequest> {
+
+    private final static HttpRequestSetter setter = new HttpRequestSetter();
+
+    public enum FieldType {
+        method,
+        port,
+        host,
+        path,
+        query,
+        header,
+        version
+    }
+
+    HttpRequest.Builder builder = HttpRequest.newBuilder();
+    Map<FieldType, StringBindings> unresolved = new HashMap<>();
+
+    // only populated if there is no value which is an actual bindings template
+    private final HttpRequest cachedRequest;
+
+    public ReadyHttpRequest(StmtDef stmtDef) {
+        CommandTemplate cmdt = new CommandTemplate(stmtDef, false);
+        ParsedStmt parsed = stmtDef.getParsed();
+
+        Map<String, String> reqParams = new HashMap<>();
+
+        String stmt = parsed.getStmt();
+        if (stmt != null) {
+            Map<String, String> parsedparams = ParamsParser.parse(stmt, false);
+            reqParams.putAll(parsedparams);
+        }
+        for (String paramsKey : stmtDef.getParams().keySet()) {
+            if (reqParams.containsKey(paramsKey)) {
+                throw new RuntimeException("request parameter '" + paramsKey + "' used again in params block. Choose one.");
+            }
+        }
+        reqParams.putAll(stmtDef.getParamsAsValueType(String.class));
+
+        for (String cfgname : reqParams.keySet()) {
+            FieldType cfgfield;
+            try {
+                cfgfield = FieldType.valueOf(cfgname);
+            } catch (IllegalArgumentException iae) {
+                throw new BasicError("You can't configure a request with '" + cfgname + "'." +
+                        " Valid properties are " + Arrays.stream(FieldType.values()).map(String::valueOf).collect(Collectors.joining(",")));
+            }
+            String value = reqParams.get(cfgname);
+            ParsedTemplate tpl = new ParsedTemplate(value, stmtDef.getBindings());
+            if (tpl.getBindPoints().size() == 0) {
+                builder = setter.setField(builder, cfgfield, value);
+            } else {
+                BindingsTemplate bindingsTemplate = new BindingsTemplate(tpl.getBindPoints());
+                StringBindingsTemplate stringBindingsTemplate = new StringBindingsTemplate(value, bindingsTemplate);
+                StringBindings stringBindings = stringBindingsTemplate.resolve();
+                unresolved.put(cfgfield, stringBindings);
+            }
+        }
+
+        if (unresolved.size() == 0) {
+            cachedRequest = builder.build();
+        } else {
+            cachedRequest = null;
+        }
+    }
+
+    @Override
+    public HttpRequest apply(long value) {
+        if (this.cachedRequest != null) {
+            return this.cachedRequest;
+        }
+        HttpRequest.Builder newRq = builder.copy();
+        for (Map.Entry<FieldType, StringBindings> toset : unresolved.entrySet()) {
+            String setValue = toset.getValue().bind(value);
+            newRq = setter.setField(newRq, toset.getKey(), setValue);
+        }
+
+        HttpRequest request = newRq.build();
+        return request;
+    }
+}

driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java

@@ -4,24 +4,38 @@ import com.codahale.metrics.Timer;
 import io.nosqlbench.engine.api.activityapi.core.SyncAction;
 import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
+import io.nosqlbench.nb.api.errors.BasicError;
 import io.nosqlbench.virtdata.core.templates.StringBindings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.net.*;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 
 public class HttpAction implements SyncAction {
 
     private final static Logger logger = LoggerFactory.getLogger(HttpAction.class);
+
     private final HttpActivity httpActivity;
     private final int slot;
     private int maxTries = 1;
     private boolean showstmts;
 
-    private OpSequence<StringBindings> sequencer;
+    private OpSequence<CommandTemplate> sequencer;
+    private HttpClient client;
+    private HttpResponse.BodyHandler<String> bodyreader = HttpResponse.BodyHandlers.ofString();
+    private long timeoutMillis;
 
 
     public HttpAction(ActivityDef activityDef, int slot, HttpActivity httpActivity) {
@@ -32,6 +46,8 @@ public class HttpAction implements SyncAction {
     @Override
     public void init() {
         this.sequencer = httpActivity.getOpSequence();
+        this.client = HttpClient.newHttpClient();
+
     }
 
     @Override
@@ -40,86 +56,179 @@ public class HttpAction implements SyncAction {
         String statement = null;
         InputStream result = null;
 
-        try (Timer.Context bindTime = httpActivity.bindTimer.time()) {
-            stringBindings = sequencer.get(cycleValue);
-            statement = stringBindings.bind(cycleValue);
+        // The bind timer captures all the time involved in preparing the
+        // operation for execution, including data generation as well as
+        // op construction
 
-            String[] splitStatement = statement.split("\\?");
-            String path, query;
+        // The request to be used must be constructed from the template each time.
+        HttpRequest request=null;
+
+        // A specifier for what makes a response ok. If this is provided, then it is
+        // either a list of valid http status codes, or if non-numeric, a regex for the body
+        // which must match.
+        // If not provided, then status code 200 is the only thing required to be matched.
+        String ok;
+
+        try (Timer.Context bindTime = httpActivity.bindTimer.time()) {
+            CommandTemplate commandTemplate = httpActivity.getOpSequence().get(cycleValue);
+            Map<String, String> cmdMap = commandTemplate.getCommand(cycleValue);
 
             String host = httpActivity.getHosts()[(int) cycleValue % httpActivity.getHosts().length];
+            String[] command = cmdMap.get("command").split(" ", 3); // RFC 2616 Section 5.1.2
+            ok = cmdMap.remove("ok");
 
-            path = splitStatement[0];
-            query = "";
+            // Base request
+            String method = command[0].toUpperCase();
 
-            if (splitStatement.length >= 2) {
-                query = splitStatement[1];
+            String baseuri = command[1].trim();
+            URI uri = URI.create(baseuri);
+
+            HttpRequest.Builder builder = HttpRequest.newBuilder(uri);
+
+            HttpRequest.BodyPublisher bodysource = bodySourceFrom(cmdMap);
+            builder = builder.method(method, bodysource);
+
+            if (command.length == 3) {
+                HttpClient.Version version = HttpClient.Version.valueOf(command[2]);
+                builder.version(version);
             }
 
-            URI uri = new URI(
-                "http",
-                null,
-                host,
-                httpActivity.getPort(),
-                path,
-                query,
-                null);
-
-                statement = uri.toString();
-
-            showstmts = httpActivity.getShowstmts();
-            if (showstmts) {
-                logger.info("STMT(cycle=" + cycleValue + "):\n" + statement);
+            // All known command options must be processed by this point, so the rest are headers
+            for (String mustBeAHeader : cmdMap.keySet()) {
+                builder.header(mustBeAHeader, cmdMap.get(mustBeAHeader));
             }
-        } catch (URISyntaxException e) {
-            e.printStackTrace();
+
+            request = builder.build();
+        } catch (Exception e) {
+            throw new RuntimeException("while binding request in cycle " + cycleValue + ": " + e.getMessage(),e);
         }
 
-        long nanoStartTime=System.nanoTime();
         int tries = 0;
-
         while (tries < maxTries) {
             tries++;
 
+            CompletableFuture<HttpResponse<String>> responseFuture;
             try (Timer.Context executeTime = httpActivity.executeTimer.time()) {
-                URL url = new URL(statement);
-//
-                HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-                conn.setRequestMethod("GET");
-                result = conn.getInputStream();
+                responseFuture = client.sendAsync(request, this.bodyreader);
             } catch (Exception e) {
-                throw new RuntimeException("Error writing output:" + e, e);
+                throw new RuntimeException("while waiting for response in cycle " + cycleValue + ":" + e.getMessage(), e);
             }
 
-            Timer.Context resultTime = httpActivity.resultTimer.time();
-            try {
-                StringBuilder res = new StringBuilder();
-
-                BufferedReader rd = new BufferedReader(new InputStreamReader(result));
-                String line;
-                while ((line = rd.readLine()) != null) {
-                    res.append(line);
-                }
-                rd.close();
-
+            HttpResponse<String> response;
+            try (Timer.Context resultTime = httpActivity.resultTimer.time()) {
+                response = responseFuture.get(timeoutMillis, TimeUnit.MILLISECONDS);
             } catch (Exception e) {
-                long resultNanos = resultTime.stop();
-                resultTime=null;
-            } finally {
-                if (resultTime!=null) {
-                    resultTime.stop();
-                }
-
+                throw new RuntimeException("while waiting for response in cycle " + cycleValue + ":" + e.getMessage(), e);
             }
 
+            if (ok == null) {
+                if (response.statusCode() != 200) {
+                    throw new ResponseError("Result had status code " +
+                            response.statusCode() + ", but 'ok' was not set for this statement," +
+                            "so it is considered an error.");
+                }
+            } else {
+                String[] oks = ok.split(",");
+                for (String ok_condition : oks) {
+                    if (ok_condition.charAt(0)>='0' && ok_condition.charAt(0)<='9') {
+                        int matching_status = Integer.parseInt(ok_condition);
+                    } else {
+                        Pattern successRegex = Pattern.compile(ok);
+                    }
+                }
+//                Matcher matcher = successRegex.matcher(String.valueOf(response.statusCode()));
+//                if (!matcher.matches()) {
+//                    throw new BasicError("status code " + response.statusCode() + " did not match " + success);
+//                }
+            }
         }
-        long resultNanos=System.nanoTime() - nanoStartTime;
-        httpActivity.resultSuccessTimer.update(resultNanos, TimeUnit.NANOSECONDS);
-
-
         return 0;
     }
-    protected HttpActivity getHttpActivity() {
-        return httpActivity;
+
+//                String body = future.body();
+
+
+//            String[] splitStatement = statement.split("\\?");
+//            String path, query;
+//
+//            path = splitStatement[0];
+//            query = "";
+//
+//            if (splitStatement.length >= 2) {
+//                query = splitStatement[1];
+//            }
+//
+//            URI uri = new URI(
+//                "http",
+//                null,
+//                host,
+//                httpActivity.getPort(),
+//                path,
+//                query,
+//                null);
+//
+//                statement = uri.toString();
+//
+//            showstmts = httpActivity.getShowstmts();
+
+//            if (showstmts) {
+//                logger.info("STMT(cycle=" + cycleValue + "):\n" + statement);
+//            }
+//        } catch (URISyntaxException e) {
+//            e.printStackTrace();
+//        }
+//
+//        long nanoStartTime=System.nanoTime();
+//
+
+//                Timer.Context resultTime = httpActivity.resultTimer.time();
+//                try {
+//                    StringBuilder res = new StringBuilder();
+//
+//                    BufferedReader rd = new BufferedReader(new InputStreamReader(result));
+//                    String line;
+//                    while ((line = rd.readLine()) != null) {
+//                        res.append(line);
+//                    }
+//                    rd.close();
+//
+//                } catch (Exception e) {
+//                    long resultNanos = resultTime.stop();
+//                    resultTime = null;
+//                } finally {
+//                    if (resultTime != null) {
+//                        resultTime.stop();
+//                    }
+//
+//                }
+//
+//            }
+//            long resultNanos = System.nanoTime() - nanoStartTime;
+//            httpActivity.resultSuccessTimer.update(resultNanos, TimeUnit.NANOSECONDS);
+
+
+//        protected HttpActivity getHttpActivity () {
+//            return httpActivity;
+//        }
+//    }
+
+    private HttpRequest.BodyPublisher bodySourceFrom(Map<String, String> cmdMap) {
+        if (cmdMap.containsKey("body")) {
+            String body = cmdMap.remove("body");
+            return HttpRequest.BodyPublishers.ofString(body);
+        } else if (cmdMap.containsKey("file")) {
+            try {
+                String file = cmdMap.get("file");
+                Path path = Path.of(file);
+                return HttpRequest.BodyPublishers.ofFile(path);
+            } catch (FileNotFoundException e) {
+                throw new BasicError("Could not find file content for request at " + cmdMap.get("file"));
+            }
+        } else {
+            return HttpRequest.BodyPublishers.noBody();
+        }
+
     }
-}
+
+
+}

driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java

@@ -15,6 +15,9 @@ import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
 import io.nosqlbench.engine.api.metrics.ActivityMetrics;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
+import io.nosqlbench.engine.api.templating.StrInterpolator;
+import io.nosqlbench.nb.api.errors.BasicError;
 import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
 import io.nosqlbench.virtdata.core.templates.StringBindings;
 import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
@@ -32,7 +35,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
         return stmtsDocList;
     }
 
-    private final StmtsDocList stmtsDocList;
+    private StmtsDocList stmtsDocList;
 
 
     private int stride;
@@ -49,17 +52,11 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
     private String[] hosts;
     private int port;
 
-    private OpSequence<StringBindings> opSequence;
+    private OpSequence<CommandTemplate> opSequence;
 
     public HttpActivity(ActivityDef activityDef) {
         super(activityDef);
         this.activityDef = activityDef;
-
-        String yaml_loc = activityDef.getParams()
-                .getOptionalString("yaml", "workload")
-                .orElse("default");
-
-        stmtsDocList = StatementsLoader.loadPath(logger,yaml_loc, "activities");
     }
 
 
@@ -76,8 +73,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
         hosts = activityDef.getParams().getOptionalString("host").orElse("localhost").split(",");
         port = activityDef.getParams().getOptionalInteger("port").orElse(80);
 
-
-        opSequence = initOpSequencer();
+        this.opSequence = createDefaultOpSequence();
         setDefaultsFromOpSequence(opSequence);
 
         bindTimer = ActivityMetrics.timer(activityDef, "bind");
@@ -91,34 +87,6 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
         onActivityDefUpdate(activityDef);
     }
 
-    private OpSequence<StringBindings> initOpSequencer() {
-        SequencerType sequencerType = SequencerType.valueOf(
-                getParams().getOptionalString("seq").orElse("bucket")
-        );
-        SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType);
-
-        String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
-        List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
-
-        if (stmts.size() > 0) {
-            for (StmtDef stmt : stmts) {
-                ParsedStmt parsed = stmt.getParsed().orError();
-                BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
-                String statement = parsed.getPositionalStatement(Function.identity());
-                Objects.requireNonNull(statement);
-
-                StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt(), bt);
-                StringBindings sb = sbt.resolve();
-                sequencer.addOp(sb,stmt.getParamOrDefault("ratio",1));
-            }
-        } else {
-            logger.error("Unable to create an HTTP statement if no bindings or statements are defined.");
-        }
-//
-        OpSequence<StringBindings> opSequence = sequencer.resolve();
-        return opSequence;
-    }
-
     @Override
     public synchronized void onActivityDefUpdate(ActivityDef activityDef) {
         super.onActivityDefUpdate(activityDef);
@@ -140,7 +108,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
         return port;
     }
 
-    public OpSequence<StringBindings> getOpSequence() {
+    public OpSequence<CommandTemplate> getOpSequence() {
         return opSequence;
     }
 }

driver-http/src/main/java/io/nosqlbench/activitytype/http/ResponseError.java

@@ -0,0 +1,6 @@
+package io.nosqlbench.activitytype.http;
+
+public class ResponseError extends RuntimeException {
+    public ResponseError(String s) {
+    }
+}

driver-http/src/main/resources/http.md

@@ -1,52 +1,84 @@
-# http activity type
+# HTTP driver
 
-This activity type allows for basic HTTP requests.
-As of this release, only GET requests are supported.
+This driver allows you to make http requests using the native HTTP client that is bundled with the
+JVM. It supports free-form construction of requests.
 
-## Example activity definitions
+## Example Statements
 
-Run an http activity named 'http-test', with definitions from activities/http-google.yaml:
-~~~
-... driver=http workload=http-google
-~~~
+You can use an _inline request template_ form below to represent a request as it would be submitted
+according to the HTTP protocol. This isn't actually the content that is submitted, but it is
+recognized as a valid way to express the request parameters in a familiar and condensed form:
 
-This last example shows that the cycle range is [inclusive..exclusive),
-to allow for stacking test intervals. This is standard across all
-activity types.
+```yaml
+statements:
+ - s1: |
+    POST http://{host}:{port}/{path}?{query} HTTP/1.1
+    Content-Type: application/json
+    token: mybearertokenfoobarbazomgwtfbbq
+    
+    {body} 
+```
 
-## stdout ActivityType Parameters
+You can also provide the building blocks of a request in named fields:
 
-- **host** - The hosts to send requests to. The hosts are selected in
-  round-robin fashion.
-  (default: localhost)
-- **workload** - The workload definition file which holds the schema and statement defs.
-  (no default, required)
-- **cycles** - standard, however the activity type will default
-  this to however many statements are included in the current
-  activity, after tag filtering, etc.
-  (default: 0)
-- **alias** - this is a standard nosqlbench parameter
-  (default: derived from the workload name)
+```yaml
+  - method: GET
+    version: HTTP/1.1
+    "Content-Type": "application/json"
+    body: {body}
+    path: {path}
+    ok-status: 2[0-9][0-9]
+    ok-body: ^(OK, account id is .*)$
+```
 
-## Configuration
+As you may guess from the above example, some reserved words are recognized as standard request
+parameters. They are explained here in more detail:
 
-This activity type uses the uniform yaml configuration format.
-For more details on this format, please refer to the
-[Standard YAML Format](http://docs.nosqlbench.io/user-guide/standard_yaml/)
+- **method** - An optional request method. If not provided, "GET" is assumed. Any method name will
+  work here, even custom ones that are specific to a given target system. No validation is done for
+  standard method names, as there is no way to know what method names may be valid.
+- **host** - The name of the host which should go into the URI. This can also be an ip address if
+  you do not need support for virtual hosts. If there are multiple hosts provided to the activity,
+  then this value is selected in round-robin style. **default: localhost**
+- **port** - The post to connect to. If it is provided, then it is added to the URI, even if it is
+  the default for the scheme (80 for http, or 443 for https)
+- **path** - The path component of the URI.
+- **query** - A query string. If this is provided, it is appended to the path in the URI with a
+  leading question mark.
+- **version** - The HTTP version to use. If this value is not provided, the default version for the
+  Java HttpClient is used. If it is provided, it must be one of 'HTTP_1_1', or 'HTTP_2'.
+- **body** - The content of the request body, for methods which support it.
+- **ok-status** - An optional set of rules to use to verify that a response is valid. This is a
+  simple comma or space separated list of integer status codes or a pattern which is used as a regex
+  against the string form of a status code. If any characters other than digits spaces and commas
+  are found in this value, then it is taken as a regex. If this is not provided, then any status
+  code which is >=200 and <300 is considered valid.
+- **ok-body** - An optional regex pattern which will be applied to the body to verify that it is a
+  valid response. If this is not provided, then content bodies are read, but any content is
+  considered valid.
 
-## Configuration Parameters
+Any other statement parameter which is capitalized is taken as a request header. If additional
+fields are provided which are not included in the above list, or which are not capitalized, then an
+error is thrown.
 
-- **ratio** - If a statement has this param defined, then it determines
-  whether or not to automatically add a missing newline for that statement
-  only. If this is not defined for a statement, then the activity-level
-  parameter takes precedence.
-- **seq** - The statement sequencer scheme.
-  (default: bucket)
+## Error Handling & Retries
 
-## Statement Format
+Presently, no determination is made about whether or not an errored response *should* be retryable.
+Contextual error handling may be added in a future version.
 
-The statement format for this activity type is a simple string. Tokens between
-curly braces are used to refer to binding names, as in the following example:
+## SSL Support
 
-    statements:
-     - "/{path}?{queryparam1}"
+SSL Support will be added before this driver is considered ready for general use.
+
+## Client Behavior
+
+### TCP Sessions
+
+The HTTP clients are allocated one to each thread. The TCP connection caching is entirely left to
+the defaults for the current HttpClient library that is bundled within the JVM.
+
+### Chunked encoding and web sockets
+
+Presently, this driver only does basic request-response style requests. Thus, adding headers which
+take TCP socket control away from the HttpClient will likely yield inconsistent (or undefined)
+results. Support may be added for long-lived connections in a future release.

driver-http/src/test/java/io/nosqlbench/activitytype/cmds/ReadyHttpRequestTest.java

@@ -0,0 +1,23 @@
+package io.nosqlbench.activitytype.cmds;
+
+import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
+import org.junit.Test;
+
+import java.net.http.HttpRequest;
+
+public class ReadyHttpRequestTest {
+
+    @Test
+    public void testStaticTemplate() {
+        StmtsDocList docs = StatementsLoader.loadString("" +
+                "statements:\n" +
+                " - s1: method=get\n");
+        StmtDef stmtDef = docs.getStmts().get(0);
+
+        ReadyHttpRequest readyReq = new ReadyHttpRequest(stmtDef);
+        HttpRequest staticReq = readyReq.apply(3);
+    }
+
+}

driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverAction.java

@@ -4,6 +4,7 @@ import io.nosqlbench.driver.webdriver.verbs.WebDriverVerbs;
 import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
 import io.nosqlbench.engine.api.activityapi.core.SyncAction;
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
 import io.nosqlbench.nb.api.errors.BasicError;
 import org.openqa.selenium.WebDriver;
 import org.slf4j.Logger;
@@ -45,6 +46,7 @@ public class WebDriverAction implements SyncAction, ActivityDefObserver {
     // factor.
     @Override
     public int runCycle(long value) {
+
         CommandTemplate commandTemplate = activity.getOpSequence().get(value);
         try {
             WebDriverVerbs.execute(value, commandTemplate, context, dryrun);

driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java

@@ -13,6 +13,7 @@ import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
 import io.nosqlbench.engine.api.templating.StrInterpolator;
 import io.nosqlbench.nb.api.content.NBIO;
 import io.nosqlbench.nb.api.errors.BasicError;

driver-web/src/main/java/io/nosqlbench/driver/webdriver/verbs/WebDriverVerbs.java

@@ -1,7 +1,7 @@
 package io.nosqlbench.driver.webdriver.verbs;
 
-import io.nosqlbench.driver.webdriver.CommandTemplate;
 import io.nosqlbench.driver.webdriver.WebContext;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
 import io.nosqlbench.nb.api.errors.BasicError;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java

@@ -22,16 +22,26 @@ import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import io.nosqlbench.nb.api.content.Content;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.function.Function;
 
 public class StatementsLoader {
 
+    private final static Logger logger = LoggerFactory.getLogger(StatementsLoader.class);
+
     public enum Loader {
         original,
         generified
     }
 
+    public static StmtsDocList loadString(String yamlContent) {
+        RawStmtsLoader loader = new RawStmtsLoader();
+        RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
+        StmtsDocList layered = new StmtsDocList(rawDocList);
+        return layered;
+    }
+
     public static StmtsDocList loadContent(
             Logger logger,
             Content<?> content) {

engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java

@@ -6,10 +6,18 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.filters.IntPredicateDispens
 import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
 import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;
 import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
+import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
+import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
 import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
 import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
 import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
+import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import io.nosqlbench.engine.api.metrics.ActivityMetrics;
+import io.nosqlbench.engine.api.templating.CommandTemplate;
+import io.nosqlbench.engine.api.templating.StrInterpolator;
+import io.nosqlbench.nb.api.errors.BasicError;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -320,4 +328,31 @@ public class SimpleActivity implements Activity {
         }
 
     }
+
+    protected OpSequence<CommandTemplate> createDefaultOpSequence() {
+        StrInterpolator interp = new StrInterpolator(activityDef);
+        String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
+        StmtsDocList stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, interp, "activities");
+
+        SequencerType sequencerType = getParams()
+                .getOptionalString("seq")
+                .map(SequencerType::valueOf)
+                .orElse(SequencerType.bucket);
+        SequencePlanner<CommandTemplate> planner = new SequencePlanner<>(sequencerType);
+
+        String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
+        List<StmtDef> stmts = stmtsDocList.getStmts(tagfilter);
+
+        if (stmts.size() == 0) {
+            throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
+        }
+
+        for (StmtDef optemplate : stmts) {
+            long ratio = optemplate.getParamOrDefault("ratio", 1);
+            CommandTemplate cmd = new CommandTemplate(optemplate, false);
+            planner.addOp(cmd, ratio);
+        }
+        return planner.resolve();
+    }
+
 }

engine-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java

@@ -1,17 +1,14 @@
-package io.nosqlbench.driver.webdriver;
+package io.nosqlbench.engine.api.templating;
 
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
 import io.nosqlbench.engine.api.activityimpl.motor.ParamsParser;
-import io.nosqlbench.nb.api.errors.BasicError;
 import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
 import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
 import io.nosqlbench.virtdata.core.templates.StringBindings;
 import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.nodes.ScalarNode;
 
-import java.security.InvalidParameterException;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -19,6 +16,14 @@ import java.util.Map;
  * Use the {@link StmtDef} template form as a property template for parameterized
  * commands. This is a general purpose template which uses a map of named parameters.
  * The {@code command} property designates the verb component of the command.
+ *
+ * To be valid for use with this template type, the template specifier (the stmt String)
+ * must either start with command= or have a single word at the start. In either case,
+ * the command will be parsed as if it started with a command=...
+ *
+ * The semantics of command are meant to be generalized. For example, with HTTP, command
+ * might mean the HTTP method like GET or PUT that is used. For web driver, it may be
+ * a webdriver command as known by the SIDE file format.
  */
 public class CommandTemplate {
 
@@ -46,7 +51,6 @@ public class CommandTemplate {
             StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
             cmdspec.put(param,paramStringBindings);
         });
-
     }
 
     public CommandTemplate(String command, Map<String,String> bindings, String name, boolean canonicalize) {
@@ -58,7 +62,6 @@ public class CommandTemplate {
             StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
             cmdspec.put(param,paramStringBindings);
         });
-
     }
 
     public Map<String,String> getCommand(long cycle) {

engine-api/src/main/java/io/nosqlbench/engine/api/templating/EnumSetter.java

@@ -0,0 +1,30 @@
+package io.nosqlbench.engine.api.templating;
+
+/**
+ * Provide a way to configure a target object of type T, given an enumeration which describes the distinct property
+ * types which could be configured. This method provides an efficient method for refining a template or builder object
+ * with O(1) field lookup.
+ * <p>
+ * The field enum doesn't limit how a field may be modified. In some cases, a single field may be iterativel built-up,
+ * such as headers for a request object. (Multiple headers can be added, and they are all a header type, just with
+ * different values.) In other cases, there may be a single-valued property that is replaced entirely each time it is
+ * set.
+ *
+ * @param <F> An enum type which describes distinct fields which may be modified.
+ * @param <T> A target configurable type to be configured.
+ */
+public interface EnumSetter<F extends Enum<F>, T> {
+
+    /**
+     * Given a target configurable of type T and a field type identifier from enum type K, set or add a value to the
+     * field described by K, and then return the target configurable.
+     *
+     * @param target The object to be augmented.
+     * @param field  The enum which describes the type of field mutation.
+     * @param value  The object values to assign to the named field. If a given field type can be a collection, like a
+     *               map, then these might be a 2-tuple of key-value. Semantics of the value objects are always
+     *               specific to the type of field being configured.
+     * @return A modified object, possibly a new instance, as with functional property modifiers.
+     */
+    T setField(T target, F field, Object... value);
+}

engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java

@@ -0,0 +1,23 @@
+package io.nosqlbench.engine.api.templating;
+
+import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtDef;
+import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CommandTemplateTest {
+
+    @Test
+    public void testCommandTemplate() {
+        StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
+                "statements:\n" +
+                " - s1: test1=foo test2=bar");
+        StmtDef stmtDef = stmtsDocs.getStmts().get(0);
+        CommandTemplate ct = new CommandTemplate(stmtDef, false);
+        assertThat(ct.isStatic()).isTrue();
+
+    }
+
+}