Merge pull request #866 from nosqlbench/nosqlbench-865-urlencode

nosqlbench-865-urlencode

adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java

@@ -20,6 +20,8 @@ import io.nosqlbench.adapter.http.core.HttpFormatParser;
 import io.nosqlbench.adapter.http.core.HttpOp;
 import io.nosqlbench.adapter.http.core.HttpOpMapper;
 import io.nosqlbench.adapter.http.core.HttpSpace;
+import io.nosqlbench.api.config.standard.ConfigModel;
+import io.nosqlbench.api.config.standard.Param;
 import io.nosqlbench.engine.api.activityimpl.OpMapper;
 import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
 import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@@ -68,6 +70,11 @@ public class HttpDriverAdapter extends BaseDriverAdapter<HttpOp, HttpSpace> {
 
     @Override
     public NBConfigModel getConfigModel() {
-        return super.getConfigModel().add(HttpSpace.getConfigModel());
+        NBConfigModel thisCfgModel = ConfigModel.of(HttpDriverAdapter.class)
+            .add(Param.defaultTo("enable_urlencode", false)
+                .setDescription("Override auto-detection of URLENCODE[[ requirements in the uri field."))
+            .asReadOnly();
+
+        return super.getConfigModel().add(HttpSpace.getConfigModel()).add(thisCfgModel);
     }
 }

adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpFormatParser.java

@@ -94,12 +94,12 @@ public class HttpFormatParser {
         return props;
     }
 
-    private final static Pattern DOENCODE = Pattern.compile("(URLENCODE|E)\\[\\[(?<data>.+?)\\]\\]");
+    public final static Pattern URLENCODER_PATTERN = Pattern.compile("(URLENCODE|E)\\[\\[(?<data>.+?)\\]\\]");
 
     public static String rewriteExplicitSections(String template) {
 
         StringBuilder sb = new StringBuilder();
-        Matcher matcher = DOENCODE.matcher(template);
+        Matcher matcher = URLENCODER_PATTERN.matcher(template);
         while (matcher.find()) {
             String rewrite = matcher.group("data");
             String encoded = rewriteStaticsOnly(rewrite);

adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpOpDispenser.java

@@ -29,7 +29,7 @@ import java.util.Optional;
 import java.util.function.LongFunction;
 import java.util.regex.Pattern;
 
-public class HttpOpDispenser extends BaseOpDispenser<HttpOp,HttpSpace> {
+public class HttpOpDispenser extends BaseOpDispenser<HttpOp, HttpSpace> {
 
     private final LongFunction<HttpOp> opFunc;
     public static final String DEFAULT_OK_BODY = ".+?";
@@ -63,9 +63,27 @@ public class HttpOpDispenser extends BaseOpDispenser<HttpOp,HttpSpace> {
             )
         );
 
-        initBuilderF = op.enhanceFuncOptionally(initBuilderF, "uri", String.class, (b, v) -> b.uri(URI.create(v)));
+        Optional<LongFunction<String>> optionalUriFunc = op.getAsOptionalFunction("uri", String.class);
+        LongFunction<String> urifunc;
+        // Add support for URLENCODE on the uri field if either it statically or dynamically contains the E or URLENCODE pattern,
+        // OR the enable_urlencode op field is set to true.
+        if (optionalUriFunc.isPresent()) {
+            String testUriValue = optionalUriFunc.get().apply(0L);
+            if (HttpFormatParser.URLENCODER_PATTERN.matcher(testUriValue).find()
+                || op.getStaticConfigOr("enable_urlencode", true)) {
+                initBuilderF =
+                    op.enhanceFuncOptionally(
+                        initBuilderF,
+                        "uri",
+                        String.class,
+                        (b, v) -> b.uri(URI.create(HttpFormatParser.rewriteExplicitSections(v)))
+                    );
+            }
+        } else {
+            initBuilderF = op.enhanceFuncOptionally(initBuilderF, "uri", String.class, (b, v) -> b.uri(URI.create(v)));
+        }
 
-        op.getOptionalStaticValue("follow_redirects",boolean.class);
+        op.getOptionalStaticValue("follow_redirects", boolean.class);
 
         /**
          * Add header adders for any key provided in the op template which is capitalized
@@ -74,19 +92,19 @@ public class HttpOpDispenser extends BaseOpDispenser<HttpOp,HttpSpace> {
             .filter(n -> n.charAt(0) >= 'A')
             .filter(n -> n.charAt(0) <= 'Z')
             .toList();
-        if (headerNames.size()>0) {
+        if (headerNames.size() > 0) {
             for (String headerName : headerNames) {
-                initBuilderF = op.enhanceFunc(initBuilderF,headerName,String.class, (b,h) -> b.header(headerName,h));
+                initBuilderF = op.enhanceFunc(initBuilderF, headerName, String.class, (b, h) -> b.header(headerName, h));
             }
         }
 
-        initBuilderF = op.enhanceFuncOptionally(initBuilderF,"timeout",long.class,(b,v) -> b.timeout(Duration.ofMillis(v)));
+        initBuilderF = op.enhanceFuncOptionally(initBuilderF, "timeout", long.class, (b, v) -> b.timeout(Duration.ofMillis(v)));
 
         LongFunction<HttpRequest.Builder> finalInitBuilderF = initBuilderF;
         LongFunction<HttpRequest> reqF = l -> finalInitBuilderF.apply(l).build();
 
 
-        Pattern ok_status = op.getOptionalStaticValue("ok-status",String.class)
+        Pattern ok_status = op.getOptionalStaticValue("ok-status", String.class)
             .map(Pattern::compile)
             .orElse(Pattern.compile(DEFAULT_OK_STATUS));
 
@@ -99,7 +117,7 @@ public class HttpOpDispenser extends BaseOpDispenser<HttpOp,HttpSpace> {
             reqF.apply(cycle),
             ok_status,
             ok_body,
-            ctxF.apply(cycle),cycle
+            ctxF.apply(cycle), cycle
         );
         return opFunc;
     }

adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java

@@ -120,7 +120,7 @@ public class HttpSpace implements NBNamedElement {
                 .setDescription("Print extended diagnostics. This option has numerous" +
                     " possible values. See the markdown docs for details. (nb help http)")
             )
-            .add(Param.defaultTo("timeout", Long.MAX_VALUE)
+            .add(Param.defaultTo("timeout", 1000L*60L*15L) // 15 minutes
                 .setDescription("How long to wait for requests before timeout out. Default is forever."))
             .add(Param.defaultTo("hdr_digits", 4)
                 .setDescription("number of digits of precision to keep in HDR histograms"))

adapter-http/src/main/resources/http.md

@@ -141,6 +141,9 @@ defaults:
   ensure that the values that are inserted at binding points are produced
   in a valid form for a URI. You can use the `URLEncode()`
   binding function where needed to achieve this.
+  *NOTE*, If you are using dynamic values for the uri field, and
+  a test value for cycle 0 includes neither `URLENCODE[[` nor `E[]`,
+  then it is skipped. You can override this with `enable_urlencode: true`.
 - **method** - An optional request method. If not provided, "GET" is
   assumed. Any method name will work here, even custom ones that are
   specific to a given target system. No validation is done for standard
@@ -187,11 +190,22 @@ configuration. If needed, more configurable SSL support will be added.
 
 ## Client Behavior
 
-### TCP Sessions
+### TCP Sessions & Clients
 
-The HTTP clients are allocated one to each thread. The TCP connection
-caching is entirely left to the defaults for the current HttpClient
-library that is bundled within the JVM.
+Client instances are created for each unique `space` value. NoSQLBench
+provides a way for all driver adapters to instance native clients according
+to a data from a binding. This is standardized under the op template parameter
+`space`, which is wired by default to the static value `default`. This means
+that each activity that uses the http driver shares a client instance across
+all threads by default. If you want to have a new http client per-thread,
+simply add a binding for `space: ThreadNumToInteger()` and reference it in
+an op template like `space: {space}`, OR use an inline op field in your op
+template like `space: {{ThreadNumToInteger()}}`.
+
+You can use any binding function you want for the space op field. However,
+if you were to assign it something like "space: {{Identity()}}" you would
+not have a good result, as you would be spinning up and caching a new http client
+instance for every single cycle.
 
 ### Chunked encoding and web sockets