IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix for #245 with always-on URL encoding of literals

Browse Source
This commit is contained in:
Jonathan Shook 2020-12-22 17:35:02 -06:00
parent f0960c4915
commit 6dc06d7ff6
8 changed files with 122 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java
View File

@ -1,21 +1,25 @@
package io.nosqlbench.activitytype.cmds;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
public class HttpFormatParser {
public static Map<String,String> parseUrl(String uri) {
public static Map<String, String> parseUrl(String uri) {
if (uri.matches("http.+")) {
return Map.of("uri",uri);
return Map.of("uri", rewriteUriWithStaticsEncoded(uri));
}
return null;
}
public static Map<String,String> parseInline(String command) {
if (command==null) {
public static Map<String, String> parseInline(String command) {
if (command == null) {
return null;
}
@ -23,41 +27,41 @@ public class HttpFormatParser {
if (!command.matches("(?m)(?s)^\\{?[a-zA-Z]+}? .+")) {
return null;
}
Map<String,String> props = new HashMap<>();
Map<String, String> props = new HashMap<>();
String[] headAndBody = command.trim().split("\n\n",2);
if (headAndBody.length==2) {
props.put("body",headAndBody[1]);
String[] headAndBody = command.trim().split("\n\n", 2);
if (headAndBody.length == 2) {
props.put("body", headAndBody[1]);
}
String[] methodAndHeaders = headAndBody[0].split("\n",2);
if (methodAndHeaders.length>1) {
String[] methodAndHeaders = headAndBody[0].split("\n", 2);
if (methodAndHeaders.length > 1) {
for (String header : methodAndHeaders[1].split("\n")) {
String[] headerNameAndVal = header.split(": *", 2);
if (headerNameAndVal.length!=2) {
if (headerNameAndVal.length != 2) {
throw new BasicError("Headers must be in 'Name: value form");
}
if (!headerNameAndVal[0].substring(0,1).toUpperCase().equals(headerNameAndVal[0].substring(0,1))) {
if (!headerNameAndVal[0].substring(0, 1).toUpperCase().equals(headerNameAndVal[0].substring(0, 1))) {
throw new BasicError("Headers must be capitalized to avoid ambiguity with other request parameters:'" + headerNameAndVal[0]);
}
props.put(headerNameAndVal[0],headerNameAndVal[1]);
props.put(headerNameAndVal[0], headerNameAndVal[1]);
}
}
String[] methodLine = methodAndHeaders[0].split(" ",3);
if (methodLine.length<2) {
String[] methodLine = methodAndHeaders[0].split(" ", 3);
if (methodLine.length < 2) {
throw new BasicError("Request template must have at least a method and a uri: " + methodAndHeaders[0]);
}
props.put("method",methodLine[0]);
props.put("uri",methodLine[1]);
props.put("method", methodLine[0]);
props.put("uri", rewriteUriWithStaticsEncoded(methodLine[1]));
if (methodLine.length==3) {
if (methodLine.length == 3) {
String actualVersion = methodLine[2];
String symbolicVersion = actualVersion
.replaceAll("/1.1","_1_1")
.replaceAll("/2.0","_2")
.replaceAll("/2","_2");
.replaceAll("/1.1", "_1_1")
.replaceAll("/2.0", "_2")
.replaceAll("/2", "_2");
props.put("version", symbolicVersion);
}
@ -65,4 +69,27 @@ public class HttpFormatParser {
return props;
}
private static String rewriteUriWithStaticsEncoded(String template) {
String[] parts = template.split("\\?", 2);
if (parts.length == 2) {
StringBuilder sb = new StringBuilder();
String input = parts[1];
Matcher matcher = ParsedTemplate.STANDARD_ANCHOR.matcher(input);
int idx = 0;
while (matcher.find()) {
String pre = input.substring(0, matcher.start());
sb.append(URLEncoder.encode(pre, StandardCharsets.UTF_8));
sb.append(matcher.group());
idx = matcher.end();
// matcher.appendReplacement(sb, "test-value" + idx);
}
sb.append(URLEncoder.encode(input.substring(idx), StandardCharsets.UTF_8));
return parts[0] + "?" + sb.toString();
} else {
return template;
}
}
}

32
driver-http/src/main/java/io/nosqlbench/activitytype/cmds/ReadyHttpOp.java
View File

@ -5,6 +5,7 @@ import io.nosqlbench.engine.api.templating.CommandTemplate;
import io.nosqlbench.nb.api.errors.BasicError;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.time.Duration;
@ -28,11 +29,35 @@ public class ReadyHttpOp implements LongFunction<HttpOp> {
)
);
sanityCheckUri();
if (propertyTemplate.isStatic()) {
cachedOp = apply(0);
} else {
cachedOp = null;
}
}
// :/?#[]@ !$&'()*+,;=
/**
* Try to catch situations in which the user put invalid characters in some part of the URI.
* In this case, the only safe thing to try seems to be to automatically urldecode
*/
private void sanityCheckUri() {
Map<String, String> command = propertyTemplate.getCommand(0L);
if (command.containsKey("uri")) {
String uriSpec = command.get("uri");
URI uri = null;
try {
uri = new URI(uriSpec);
} catch (URISyntaxException e) {
throw new BasicError(e.getMessage() + ", either use URLEncode in your bindings for values which could " +
"contain invalid URI characters, or modify the static portions of your op template to use the" +
" appropriate encodings.");
}
}
}
@Override
@ -95,4 +120,11 @@ public class ReadyHttpOp implements LongFunction<HttpOp> {
return new HttpOp(request, ok_status, ok_body);
}
@Override
public String toString() {
return "ReadyHttpOp{" +
"template=" + propertyTemplate +
", cachedOp=" + cachedOp +
'}';
}
}

6
driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java
View File

@ -10,8 +10,8 @@ import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.net.http.HttpClient;
import java.util.function.Function;
@ -54,6 +54,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
skippedTokens = ActivityMetrics.histogram(activityDef, "skipped-tokens");
resultSuccessTimer = ActivityMetrics.timer(activityDef, "result-success");
this.sequencer = createOpSequence(ReadyHttpOp::new);
setDefaultsFromOpSequence(sequencer);
onActivityDefUpdate(activityDef);
}
@ -97,7 +98,8 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
this.activityClient = newClient();
}
return t -> this.activityClient;
default: throw new RuntimeException("unable to recoginize client scope: " + getClientScope());
default:
throw new RuntimeException("unable to recoginize client scope: " + getClientScope());
}
}

11
driver-http/src/main/resources/http.md
View File

@ -117,8 +117,15 @@ faster, since the request is fully built and cached at startup.
At a minimum, a **URI** must be provided. These are enough to build a request with.
All other request fields are optional and have reasonable defaults:
- **uri** - This is the URI that you might put into the URL bar of your browser. There is no
default. Example: `https://en.wikipedia.org/wiki/Leonhard_Euler`
- **uri** - This is the URI that you might put into the URL bar of your
browser. There is no default.
Example: `https://en.wikipedia.org/wiki/Leonhard_Euler`
If the uri contains a question mark '?' as a query delimiter, then all
characters after this are automatically URL encoded. This is done for
any literal part of the uri. If you use bindings in the uri as
in `https://en.wikipedia.org/wiki/{topic}`, then it is up to you to
ensure that the values are produced in a valid form for a URI. You can
use the `URLEncode()` binding function where needed to achieve this.
- **method** - An optional request method. If not provided, "GET" is assumed. Any method name will
work here, even custom ones that are specific to a given target system. No validation is done for
standard method names, as there is no way to know what method names may be valid.

8
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/planning/SequencePlanner.java
View File

@ -63,4 +63,12 @@ public class SequencePlanner<T> {
return new Sequence<>(sequencerType, elements, elementIndex);
}
public List<T> getElements() {
return elements;
}
public List<Long> getRatios() {
return ratios;
}
}

9
engine-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java
View File

@ -175,4 +175,13 @@ public class CommandTemplate {
return this.statics.keySet();
}
@Override
public String toString() {
return "CommandTemplate{" +
"name='" + name + '\'' +
", statics=" + statics +
", dynamics=" + dynamics +
'}';
}
}

13
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java
View File

@ -9,7 +9,7 @@ import io.nosqlbench.virtdata.core.bindings.Bindings;
public class StringBindings implements Binder<String> {
private final StringCompositor compositor;
private Bindings bindings;
private final Bindings bindings;
public StringBindings(StringCompositor compositor, Bindings bindings) {
this.compositor = compositor;
@ -18,12 +18,21 @@ public class StringBindings implements Binder<String> {
/**
* Call the data mapper bindings, assigning the returned values positionally to the anchors in the string binding.
*
* @param value a long input value
* @return a new String containing the mapped values
*/
@Override
public String bind(long value) {
String s = compositor.bindValues(compositor,bindings,value);
String s = compositor.bindValues(compositor, bindings, value);
return s;
}
@Override
public String toString() {
return "StringBindings{" +
"compositor=" + compositor +
", bindings=" + bindings +
'}';
}
}

61
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java
View File

@ -18,8 +18,7 @@ import java.util.function.Function;
*/
public class StringCompositor implements ValuesBinder<StringCompositor, String> {
// protected static Pattern tokenPattern = Pattern.compile("(?<section>(?<literal>([^{])+)?(?<anchor>\\{(?<token>[a-zA-Z0-9-_.]+)?\\})?)");
private String[] templateSegments;
private final String[] templateSegments;
private Function<Object, String> stringfunc = String::valueOf;
/**
@ -42,64 +41,6 @@ public class StringCompositor implements ValuesBinder<StringCompositor, String>
return parsed.getSpans();
}
// // for testing
// protected String[] parseSection(String template) {
// StringBuilder literalBuf = new StringBuilder();
// int i = 0;
// for (; i < template.length(); i++) {
// char c = template.charAt(i);
// if (c == '\\') {
// i++;
// c = template.charAt(i);
// literalBuf.append(c);
// } else if (c != '{') {
// literalBuf.append(c);
// } else {
// i++;
// break;
// }
// }
// StringBuilder tokenBuf = new StringBuilder();
// for (; i < template.length(); i++) {
// char c = template.charAt(i);
// if (c != '}') {
// tokenBuf.append(c);
// } else {
// i++;
// break;
// }
// }
// String literal=literalBuf.toString();
// String token = tokenBuf.toString();
// if (token.length()>0) {
// return new String[] { literalBuf.toString(), tokenBuf.toString(), template.substring(i)};
// } else {
// return new String[] { literalBuf.toString() };
// }
// }
//
// /**
// * Parse the template according to the description for {@link StringCompositor}.
// *
// * @param template A string template.
// * @return A template array.
// */
// protected String[] parseTemplate(String template) {
// List<String> sections = new ArrayList<>();
//
// String[] parts = parseSection(template);
// while (parts.length>0) {
// sections.add(parts[0]);
// if (parts.length>1) {
// sections.add(parts[1]);
// }
// parts = parts.length>=2 ? parseSection(parts[2]) : new String[0];
// }
// if ((sections.size() % 2) == 0) {
// sections.add("");
// }
// return sections.toArray(new String[0]);
// }
@Override
public String bindValues(StringCompositor context, Bindings bindings, long cycle) {