IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Handle nested parantheses in TEMPLATE processing (#2072)

Browse Source 
* Handle nested parantheses in TEMPLATE processing

* Add more complex template to scenario

* Debugging and other advisors

* Tab cleanup

* Add double nesting fix and test
This commit is contained in:
Dave Fisher
2024-11-12 11:42:07 -08:00
committed by GitHub
parent 7a9b2237de
commit 64917a2879
6 changed files with 181 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@@ -20,6 +20,7 @@ import com.amazonaws.util.StringInputStream;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
import io.nosqlbench.nb.api.advisor.NBAdvisorException;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList;
@@ -29,6 +30,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.adapters.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Level;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import scala.Option;
@@ -69,6 +71,9 @@ public class OpsLoader {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
NBAdvisorOutput.render(Level.INFO,"Transform:");
NBAdvisorOutput.render(Level.INFO,"From: "+sourceData);
NBAdvisorOutput.render(Level.INFO," To: "+data);
if (srcuri!=null) {
logger.info("workload URI: '" + srcuri + "'");
}

70
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@@ -17,10 +17,12 @@
package io.nosqlbench.adapters.api.templating;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
import org.apache.commons.text.StrLookup;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Level;
import java.util.*;
import java.util.function.Function;
@@ -35,12 +37,6 @@ public class StrInterpolator implements Function<String, String> {
.setEnableUndefinedVariableException(true)
.setDisableSubstitutionInValues(true);
private final StringSubstitutor substitutor2 =
new StringSubstitutor(multimap, "TEMPLATE(", ")", '\\')
.setEnableSubstitutionInVariables(true)
.setEnableUndefinedVariableException(true)
.setDisableSubstitutionInValues(true);
public StrInterpolator(ActivityDef... activityDefs) {
Arrays.stream(activityDefs)
.map(ad -> ad.getParams().getStringStringMap())
@@ -58,10 +54,23 @@ public class StrInterpolator implements Function<String, String> {
@Override
public String apply(String raw) {
String after = substitutor.replace(substitutor2.replace(raw));
String after = matchTemplates(raw);
while (!after.equals(raw)) {
raw = after;
after = substitutor.replace(substitutor2.replace(raw));
after = matchTemplates(raw);
}
raw = after;
String original = raw;
after = substitutor.replace(raw);
while (!after.equals(raw)) {
raw = after;
after = substitutor.replace(raw);
}
if ( !original.equals(after)) {
NBAdvisorOutput.output(Level.WARN,"Transform <<key:value>>");
NBAdvisorOutput.output(Level.WARN,"From: "+original);
NBAdvisorOutput.output(Level.WARN," To: "+after);
NBAdvisorOutput.test("Using the deprecated template for of <<key:value>> please use TEMPLATE(key,value)");
}
return after;
}
@@ -76,6 +85,50 @@ public class StrInterpolator implements Function<String, String> {
return details;
}
public String matchTemplates(String original) {
String line = original;
int length = line.length();
int i = 0;
while (i < length) {
// Detect an instance of "TEMPLATE("
if (line.startsWith("TEMPLATE(", i)) {
int start = i + "TEMPLATE(".length();
int openParensCount = 1; // We found one '(' with "TEMPLATE("
// Find the corresponding closing ')' for this TEMPLATE instance
int j = start;
int k = start;
while (j < length && openParensCount > 0) {
if (line.charAt(j) == '(') {
openParensCount++;
} else if (line.charAt(j) == ')') {
k = j;
openParensCount--;
}
j++;
}
// check for case of not enough ')'
if (openParensCount > 0 ) {
if ( k != start ) {
j = k + 1;
}
openParensCount = 0;
}
// `j` now points just after the closing ')' of this TEMPLATE
if (openParensCount == 0) {
String templateContent = line.substring(start, j - 1);
// Resolve the template content
String resolvedContent = multimap.lookup(templateContent);
line = line.substring(0, i) + resolvedContent + line.substring(j);
// Update `length` and `i` based on the modified `line`
// i += resolvedContent.length() - 1;
length = line.length();
}
}
i++;
}
return line;
}
public static class MultiMap extends StrLookup<String> {
private final List<Map<String, ?>> maps = new ArrayList<>();
@@ -132,5 +185,4 @@ public class StrInterpolator implements Function<String, String> {
}
}
}

28
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java
View File

@@ -19,6 +19,7 @@ package io.nosqlbench.adapters.api.templating;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -77,6 +78,13 @@ public class StrInterpolatorTest {
assertThat(b).isEqualTo("value2");
}
@Test
public void shouldMatchNestedParens() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("TEMPLATE(keydist,Uniform(0,1000000000)->int);");
assertThat(a).isEqualTo("Uniform(0,1000000000)->int;");
}
@Test
public void shouldReturnWarningWhenUnmatched() {
StrInterpolator interp = new StrInterpolator(abcd);
@@ -129,20 +137,22 @@ public class StrInterpolatorTest {
assertThat(a).isEqualTo("'Key': 'Value'.'Stuff'");
}
// @Test
// public void shouldExpandNestedTemplates() {
// String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
// assertThat(a).isEqualTo("-aval1-");
// String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-");
// assertThat(b).isEqualTo("-bval1-");
// }
//
@Test
public void shouldExpandNestedTemplates() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
assertThat(a).isEqualTo("-aval1-");
String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-");
assertThat(b).isEqualTo("-bval1-");
}
// @Test
// public void shouldGetBasicDetails() {
// StrInterpolator interp = new StrInterpolator(abcd);
// LinkedHashMap<String, String> details = interp.getTemplateDetails("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
// assertThat(details).containsOnlyKeys("akey","dkey");
// assertThat(details).containsValues("test1");
//
// }
//
}