fix for #245 with always-on URL encoding of literals

driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java

@@ -1,15 +1,19 @@
 package io.nosqlbench.activitytype.cmds;
 
 import io.nosqlbench.nb.api.errors.BasicError;
+import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
 
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.regex.Matcher;
 
 public class HttpFormatParser {
 
     public static Map<String, String> parseUrl(String uri) {
         if (uri.matches("http.+")) {
-            return Map.of("uri",uri);
+            return Map.of("uri", rewriteUriWithStaticsEncoded(uri));
         }
         return null;
     }
@@ -50,7 +54,7 @@ public class HttpFormatParser {
             throw new BasicError("Request template must have at least a method and a uri: " + methodAndHeaders[0]);
         }
         props.put("method", methodLine[0]);
-        props.put("uri",methodLine[1]);
+        props.put("uri", rewriteUriWithStaticsEncoded(methodLine[1]));
 
         if (methodLine.length == 3) {
             String actualVersion = methodLine[2];
@@ -65,4 +69,27 @@ public class HttpFormatParser {
         return props;
     }
 
+    private static String rewriteUriWithStaticsEncoded(String template) {
+        String[] parts = template.split("\\?", 2);
+
+        if (parts.length == 2) {
+            StringBuilder sb = new StringBuilder();
+            String input = parts[1];
+            Matcher matcher = ParsedTemplate.STANDARD_ANCHOR.matcher(input);
+            int idx = 0;
+            while (matcher.find()) {
+                String pre = input.substring(0, matcher.start());
+                sb.append(URLEncoder.encode(pre, StandardCharsets.UTF_8));
+                sb.append(matcher.group());
+                idx = matcher.end();
+//                matcher.appendReplacement(sb, "test-value" + idx);
+            }
+            sb.append(URLEncoder.encode(input.substring(idx), StandardCharsets.UTF_8));
+            return parts[0] + "?" + sb.toString();
+        } else {
+            return template;
+        }
+
+    }
+
 }

driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpOp.java

@@ -5,6 +5,7 @@ import io.nosqlbench.engine.api.templating.CommandTemplate;
 import io.nosqlbench.nb.api.errors.BasicError;
 
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.time.Duration;
@@ -28,11 +29,35 @@ public class ReadyHttpOp implements LongFunction<HttpOp> {
                 )
         );
 
+        sanityCheckUri();
         if (propertyTemplate.isStatic()) {
             cachedOp = apply(0);
         } else {
             cachedOp = null;
         }
+
+    }
+
+    // :/?#[]@ !$&'()*+,;=
+
+    /**
+     * Try to catch situations in which the user put invalid characters in some part of the URI.
+     * In this case, the only safe thing to try seems to be to automatically urldecode
+     */
+    private void sanityCheckUri() {
+        Map<String, String> command = propertyTemplate.getCommand(0L);
+        if (command.containsKey("uri")) {
+            String uriSpec = command.get("uri");
+            URI uri = null;
+            try {
+                uri = new URI(uriSpec);
+            } catch (URISyntaxException e) {
+                throw new BasicError(e.getMessage() + ", either use URLEncode in your bindings for values which could " +
+                        "contain invalid URI characters, or modify the static portions of your op template to use the" +
+                        " appropriate encodings.");
+            }
+        }
+
     }
 
     @Override
@@ -95,4 +120,11 @@ public class ReadyHttpOp implements LongFunction<HttpOp> {
         return new HttpOp(request, ok_status, ok_body);
     }
 
+    @Override
+    public String toString() {
+        return "ReadyHttpOp{" +
+                "template=" + propertyTemplate +
+                ", cachedOp=" + cachedOp +
+                '}';
+    }
 }

driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java

@@ -10,8 +10,8 @@ import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
 import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
 import io.nosqlbench.engine.api.metrics.ActivityMetrics;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.net.http.HttpClient;
 import java.util.function.Function;
@@ -54,6 +54,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
         skippedTokens = ActivityMetrics.histogram(activityDef, "skipped-tokens");
         resultSuccessTimer = ActivityMetrics.timer(activityDef, "result-success");
         this.sequencer = createOpSequence(ReadyHttpOp::new);
+
         setDefaultsFromOpSequence(sequencer);
         onActivityDefUpdate(activityDef);
     }
@@ -97,7 +98,8 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
                     this.activityClient = newClient();
                 }
                 return t -> this.activityClient;
-            default: throw new RuntimeException("unable to recoginize client scope: " + getClientScope());
+            default:
+                throw new RuntimeException("unable to recoginize client scope: " + getClientScope());
         }
     }
 

driver-http/src/main/resources/http.md

@@ -117,8 +117,15 @@ faster, since the request is fully built and cached at startup.
 At a minimum, a **URI** must be provided. These are enough to build a request with.
 All other request fields are optional and have reasonable defaults:
 
-- **uri** - This is the URI that you might put into the URL bar of your browser. There is no
+- **uri** - This is the URI that you might put into the URL bar of your
-  default. Example: `https://en.wikipedia.org/wiki/Leonhard_Euler`
+  browser. There is no default.
+  Example: `https://en.wikipedia.org/wiki/Leonhard_Euler`
+  If the uri contains a question mark '?' as a query delimiter, then all
+  characters after this are automatically URL encoded. This is done for
+  any literal part of the uri. If you use bindings in the uri as
+  in `https://en.wikipedia.org/wiki/{topic}`, then it is up to you to
+  ensure that the values are produced in a valid form for a URI. You can
+  use the `URLEncode()` binding function where needed to achieve this.
 - **method** - An optional request method. If not provided, "GET" is assumed. Any method name will
   work here, even custom ones that are specific to a given target system. No validation is done for
   standard method names, as there is no way to know what method names may be valid.

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/planning/SequencePlanner.java

@@ -63,4 +63,12 @@ public class SequencePlanner<T> {
         return new Sequence<>(sequencerType, elements, elementIndex);
     }
 
+    public List<T> getElements() {
+        return elements;
+    }
+
+    public List<Long> getRatios() {
+        return ratios;
+    }
+
 }

engine-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java

@@ -175,4 +175,13 @@ public class CommandTemplate {
         return this.statics.keySet();
     }
 
+
+    @Override
+    public String toString() {
+        return "CommandTemplate{" +
+                "name='" + name + '\'' +
+                ", statics=" + statics +
+                ", dynamics=" + dynamics +
+                '}';
+    }
 }

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java

@@ -9,7 +9,7 @@ import io.nosqlbench.virtdata.core.bindings.Bindings;
 public class StringBindings implements Binder<String> {
 
     private final StringCompositor compositor;
-    private Bindings bindings;
+    private final Bindings bindings;
 
     public StringBindings(StringCompositor compositor, Bindings bindings) {
         this.compositor = compositor;
@@ -18,6 +18,7 @@ public class StringBindings implements Binder<String> {
 
     /**
      * Call the data mapper bindings, assigning the returned values positionally to the anchors in the string binding.
+     *
      * @param value a long input value
      * @return a new String containing the mapped values
      */
@@ -26,4 +27,12 @@ public class StringBindings implements Binder<String> {
         String s = compositor.bindValues(compositor, bindings, value);
         return s;
     }
+
+    @Override
+    public String toString() {
+        return "StringBindings{" +
+                "compositor=" + compositor +
+                ", bindings=" + bindings +
+                '}';
+    }
 }

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java

@@ -18,8 +18,7 @@ import java.util.function.Function;
  */
 public class StringCompositor implements ValuesBinder<StringCompositor, String> {
 
-//    protected static Pattern tokenPattern = Pattern.compile("(?<section>(?<literal>([^{])+)?(?<anchor>\\{(?<token>[a-zA-Z0-9-_.]+)?\\})?)");
+    private final String[] templateSegments;
-    private String[] templateSegments;
     private Function<Object, String> stringfunc = String::valueOf;
 
     /**
@@ -42,64 +41,6 @@ public class StringCompositor implements ValuesBinder<StringCompositor, String>
         return parsed.getSpans();
     }
 
-//    // for testing
-//    protected String[] parseSection(String template) {
-//        StringBuilder literalBuf = new StringBuilder();
-//        int i = 0;
-//        for (; i < template.length(); i++) {
-//            char c = template.charAt(i);
-//            if (c == '\\') {
-//                i++;
-//                c = template.charAt(i);
-//                literalBuf.append(c);
-//            } else if (c != '{') {
-//                literalBuf.append(c);
-//            } else  {
-//                i++;
-//                break;
-//            }
-//        }
-//        StringBuilder tokenBuf = new StringBuilder();
-//        for (; i < template.length(); i++) {
-//            char c = template.charAt(i);
-//            if (c != '}') {
-//                tokenBuf.append(c);
-//            } else {
-//                i++;
-//                break;
-//            }
-//        }
-//        String literal=literalBuf.toString();
-//        String token = tokenBuf.toString();
-//        if (token.length()>0) {
-//            return new String[] { literalBuf.toString(), tokenBuf.toString(), template.substring(i)};
-//        } else {
-//            return new String[] { literalBuf.toString() };
-//        }
-//    }
-//
-//    /**
-//     * Parse the template according to the description for {@link StringCompositor}.
-//     *
-//     * @param template A string template.
-//     * @return A template array.
-//     */
-//    protected String[] parseTemplate(String template) {
-//        List<String> sections = new ArrayList<>();
-//
-//        String[] parts = parseSection(template);
-//        while (parts.length>0) {
-//            sections.add(parts[0]);
-//            if (parts.length>1) {
-//                sections.add(parts[1]);
-//            }
-//            parts = parts.length>=2 ? parseSection(parts[2]) : new String[0];
-//        }
-//        if ((sections.size() % 2) == 0) {
-//            sections.add("");
-//        }
-//        return sections.toArray(new String[0]);
-//    }
 
     @Override
     public String bindValues(StringCompositor context, Bindings bindings, long cycle) {