IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Refactor StrInterpolator including Skipping Comments and Adding Support for Bash Variable Substitution (#2092)

Browse Source 
* Commented TEMPLATEs in yamls are ignored

* Templates only processed in StrInterpolator

* Improve ends with new line test

* Adjust deprecated template advisor message

* Recursively process template key-values

* Adds bash style simple templates

---------

Co-authored-by: Jonathan Shook <jshook@users.noreply.github.com>
This commit is contained in:
Dave Fisher 2024-11-21 09:59:23 -08:00 committed by GitHub
parent 730c3a04d5
commit da609d76da
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 95 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@ -20,7 +20,6 @@ import com.amazonaws.util.StringInputStream;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
import io.nosqlbench.nb.api.advisor.NBAdvisorException;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList;
@ -69,14 +68,11 @@ public class OpsLoader {
public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
NBAdvisorOutput.render(Level.INFO,"Transform:");
NBAdvisorOutput.render(Level.INFO,"From: "+sourceData);
NBAdvisorOutput.render(Level.INFO," To: "+data);
if (srcuri!=null) {
logger.info("workload URI: '" + srcuri + "'");
}
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
RawOpsLoader loader = new RawOpsLoader(transformer);
RawOpsDocList rawOpsDocList = switch (fmt) {

83
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@ -54,25 +54,23 @@ public class StrInterpolator implements Function<String, String> {
@Override
public String apply(String raw) {
String after = matchTemplates(raw);
while (!after.equals(raw)) {
raw = after;
after = matchTemplates(raw);
String[] lines = raw.split("\\R");
boolean endsWithNewline = raw.endsWith("\n");
int i = 0;
for (String line : lines) {
if (!line.startsWith("#")) {
String result = matchTemplates(line);
if (!result.equals(line)) {
lines[i] = result;
}
}
i++;
}
raw = after;
String original = raw;
after = substitutor.replace(raw);
while (!after.equals(raw)) {
raw = after;
after = substitutor.replace(raw);
String results = String.join(System.lineSeparator(), lines);
if (endsWithNewline) {
results += System.lineSeparator();
}
if ( !original.equals(after)) {
NBAdvisorOutput.render(Level.WARN,"Transform <<key:value>>");
NBAdvisorOutput.render(Level.WARN,"From: "+original);
NBAdvisorOutput.render(Level.WARN," To: "+after);
NBAdvisorOutput.render(Level.WARN,"The deprecated template <<key:value>> in use. Please use TEMPLATE(key,value)");
}
return after;
return results;
}
public Map<String,String> checkpointAccesses() {
@ -86,12 +84,44 @@ public class StrInterpolator implements Function<String, String> {
}
public String matchTemplates(String original) {
// process TEMPLATE(...)
String line = original;
int length = line.length();
int i = 0;
while (i < length) {
// Detect an instance of "TEMPLATE("
if (line.startsWith("TEMPLATE(", i)) {
if (line.startsWith("${", i)) {
int start = i + "${".length();
int openParensCount = 1; // We found one '{' with "${"
// Find the corresponding closing ')' for this TEMPLATE instance
int j = start;
int k = start;
while (j < length && openParensCount > 0) {
if (line.charAt(j) == '{') {
openParensCount++;
} else if (line.charAt(j) == '}') {
k = j;
openParensCount--;
}
j++;
}
// check for case of not enough '}'
if (openParensCount > 0 ) {
if ( k != start ) {
j = k + 1;
}
openParensCount = 0;
}
// `j` now points just after the closing '}' of this ${
if (openParensCount == 0) {
String templateContent = line.substring(start, j - 1);
// Recursively process
String templateValue = matchTemplates(templateContent);
String resolvedContent = multimap.lookup(templateValue);
line = line.substring(0, i) + resolvedContent + line.substring(j);
length = line.length();
i--;
}
} else if (line.startsWith("TEMPLATE(", i)) {
int start = i + "TEMPLATE(".length();
int openParensCount = 1; // We found one '(' with "TEMPLATE("
// Find the corresponding closing ')' for this TEMPLATE instance
@ -116,16 +146,23 @@ public class StrInterpolator implements Function<String, String> {
// `j` now points just after the closing ')' of this TEMPLATE
if (openParensCount == 0) {
String templateContent = line.substring(start, j - 1);
// Resolve the template content
String resolvedContent = multimap.lookup(templateContent);
// Recursively process
String templateValue = matchTemplates(templateContent);
String resolvedContent = multimap.lookup(templateValue);
line = line.substring(0, i) + resolvedContent + line.substring(j);
// Update `length` and `i` based on the modified `line`
// i += resolvedContent.length() - 1;
length = line.length();
i--;
}
}
i++;
}
// Process << ... >>
String after = substitutor.replace(line);
while (!after.equals(line)) {
NBAdvisorOutput.test("<<key:value>> deprecated in "+line);
line = after;
after = substitutor.replace(line);
}
return line;
}

30
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java
View File

@ -78,6 +78,15 @@ public class StrInterpolatorTest {
assertThat(b).isEqualTo("value2");
}
@Test
public void shouldMatchNewAlternateSubst() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("${akey}");
assertThat(a).isEqualTo("aval1");
String b = interp.apply("${nokeymatches:value2}");
assertThat(b).isEqualTo("value2");
}
@Test
public void shouldMatchNestedParens() {
StrInterpolator interp = new StrInterpolator(abcd);
@ -85,6 +94,27 @@ public class StrInterpolatorTest {
assertThat(a).isEqualTo("Uniform(0,1000000000)->int;");
}
@Test
public void shouldMatchNestedParens2() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("${keydist:Uniform(0,1000000000)->int};");
assertThat(a).isEqualTo("Uniform(0,1000000000)->int;");
}
@Test
public void shouldMatchNestedParens3() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("${keydist:${nokeymatchesthis2:Uniform(0,1000000000)->int}};");
assertThat(a).isEqualTo("Uniform(0,1000000000)->int;");
}
@Test
public void shouldMatchWithComments() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("TEMPLATE(start,START)\n# TEMPLATE(blahblah,blah)\nTEMPLATE(keydist,Uniform(0,1000000000)->int);");
assertThat(a).isEqualTo("START\n# TEMPLATE(blahblah,blah)\nUniform(0,1000000000)->int;");
}
@Test
public void shouldReturnWarningWhenUnmatched() {
StrInterpolator interp = new StrInterpolator(abcd);

69
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java
View File

@ -350,11 +350,10 @@ public class NBCLIScenarioPreprocessor {
}
Map<String, String> templates = new LinkedHashMap<>();
StrInterpolator templateparser = new StrInterpolator(templates);
try {
List<String> lines = Files.readAllLines(yamlPath);
for (String line : lines) {
templates = matchTemplates(line, templates);
}
templateparser.apply( Files.readString(yamlPath) );
templates = templateparser.checkpointAccesses();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -433,66 +432,4 @@ public class NBCLIScenarioPreprocessor {
}
// private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
// private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static Map<String, String> matchTemplates(String line, Map<String, String> templates) {
int length = line.length();
int i = 0;
while (i < length) {
// Detect an instance of "TEMPLATE("
if (line.startsWith("TEMPLATE(", i)) {
int start = i + "TEMPLATE(".length();
int openParensCount = 1; // We found one '(' with "TEMPLATE("
// Find the corresponding closing ')' for this TEMPLATE instance
int j = start;
while (j < length && openParensCount > 0) {
if (line.charAt(j) == '(') {
openParensCount++;
} else if (line.charAt(j) == ')') {
openParensCount--;
}
j++;
}
// `j` now points just after the closing ')' of this TEMPLATE
if (openParensCount == 0) {
String templateContent = line.substring(start, j - 1);
// Resolve the template content
String resolvedContent = resolveTemplate(templateContent, templates);
line = line.substring(0, i) + resolvedContent + line.substring(j);
// Update `length` and `i` based on the modified `line`
i += resolvedContent.length() - 1;
length = line.length();
}
}
i++;
}
Matcher matcher = templatePattern2.matcher(line);
while (matcher.find()) {
String match = matcher.group(1);
String[] matchArray = match.split(":");
if (matchArray.length == 1) {
templates.put(matchArray[0], "-none-");
} else {
templates.put(matchArray[0], matchArray[1]);
}
}
return templates;
}
private static String resolveTemplate(String content, Map<String, String> templates) {
String[] parts = content.split("[,:]", 2);
String key = parts[0];
String value = parts.length > 1 ? parts[1] : "";
// Store or retrieve the resolved value for the template key
if (!templates.containsKey(key)) {
templates.put(key, value);
}
return templates.getOrDefault(key, "");
}
}