IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-24 09:40:30 -06:00
Code Issues Packages Projects Releases Wiki Activity

Handle nested parantheses in TEMPLATE processing (#2072)

Browse Source 
* Handle nested parantheses in TEMPLATE processing

* Add more complex template to scenario

* Debugging and other advisors

* Tab cleanup

* Add double nesting fix and test
This commit is contained in:
Dave Fisher 2024-11-12 11:42:07 -08:00 committed by GitHub
parent 7a9b2237de
commit 64917a2879
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 181 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@ -20,6 +20,7 @@ import com.amazonaws.util.StringInputStream;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
import io.nosqlbench.nb.api.advisor.NBAdvisorException;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList;
@ -29,6 +30,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.adapters.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Level;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import scala.Option;
@ -69,6 +71,9 @@ public class OpsLoader {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
NBAdvisorOutput.render(Level.INFO,"Transform:");
NBAdvisorOutput.render(Level.INFO,"From: "+sourceData);
NBAdvisorOutput.render(Level.INFO," To: "+data);
if (srcuri!=null) {
logger.info("workload URI: '" + srcuri + "'");
}

70
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java
View File

@ -17,10 +17,12 @@
package io.nosqlbench.adapters.api.templating;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
import org.apache.commons.text.StrLookup;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Level;
import java.util.*;
import java.util.function.Function;
@ -35,12 +37,6 @@ public class StrInterpolator implements Function<String, String> {
.setEnableUndefinedVariableException(true)
.setDisableSubstitutionInValues(true);
private final StringSubstitutor substitutor2 =
new StringSubstitutor(multimap, "TEMPLATE(", ")", '\\')
.setEnableSubstitutionInVariables(true)
.setEnableUndefinedVariableException(true)
.setDisableSubstitutionInValues(true);
public StrInterpolator(ActivityDef... activityDefs) {
Arrays.stream(activityDefs)
.map(ad -> ad.getParams().getStringStringMap())
@ -58,10 +54,23 @@ public class StrInterpolator implements Function<String, String> {
@Override
public String apply(String raw) {
String after = substitutor.replace(substitutor2.replace(raw));
String after = matchTemplates(raw);
while (!after.equals(raw)) {
raw = after;
after = substitutor.replace(substitutor2.replace(raw));
after = matchTemplates(raw);
}
raw = after;
String original = raw;
after = substitutor.replace(raw);
while (!after.equals(raw)) {
raw = after;
after = substitutor.replace(raw);
}
if ( !original.equals(after)) {
NBAdvisorOutput.output(Level.WARN,"Transform <<key:value>>");
NBAdvisorOutput.output(Level.WARN,"From: "+original);
NBAdvisorOutput.output(Level.WARN," To: "+after);
NBAdvisorOutput.test("Using the deprecated template for of <<key:value>> please use TEMPLATE(key,value)");
}
return after;
}
@ -76,6 +85,50 @@ public class StrInterpolator implements Function<String, String> {
return details;
}
public String matchTemplates(String original) {
String line = original;
int length = line.length();
int i = 0;
while (i < length) {
// Detect an instance of "TEMPLATE("
if (line.startsWith("TEMPLATE(", i)) {
int start = i + "TEMPLATE(".length();
int openParensCount = 1; // We found one '(' with "TEMPLATE("
// Find the corresponding closing ')' for this TEMPLATE instance
int j = start;
int k = start;
while (j < length && openParensCount > 0) {
if (line.charAt(j) == '(') {
openParensCount++;
} else if (line.charAt(j) == ')') {
k = j;
openParensCount--;
}
j++;
}
// check for case of not enough ')'
if (openParensCount > 0 ) {
if ( k != start ) {
j = k + 1;
}
openParensCount = 0;
}
// `j` now points just after the closing ')' of this TEMPLATE
if (openParensCount == 0) {
String templateContent = line.substring(start, j - 1);
// Resolve the template content
String resolvedContent = multimap.lookup(templateContent);
line = line.substring(0, i) + resolvedContent + line.substring(j);
// Update `length` and `i` based on the modified `line`
// i += resolvedContent.length() - 1;
length = line.length();
}
}
i++;
}
return line;
}
public static class MultiMap extends StrLookup<String> {
private final List<Map<String, ?>> maps = new ArrayList<>();
@ -132,5 +185,4 @@ public class StrInterpolator implements Function<String, String> {
}
}
}

28
nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java
View File

@ -19,6 +19,7 @@ package io.nosqlbench.adapters.api.templating;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -77,6 +78,13 @@ public class StrInterpolatorTest {
assertThat(b).isEqualTo("value2");
}
@Test
public void shouldMatchNestedParens() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("TEMPLATE(keydist,Uniform(0,1000000000)->int);");
assertThat(a).isEqualTo("Uniform(0,1000000000)->int;");
}
@Test
public void shouldReturnWarningWhenUnmatched() {
StrInterpolator interp = new StrInterpolator(abcd);
@ -129,20 +137,22 @@ public class StrInterpolatorTest {
assertThat(a).isEqualTo("'Key': 'Value'.'Stuff'");
}
// @Test
// public void shouldExpandNestedTemplates() {
// String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
// assertThat(a).isEqualTo("-aval1-");
// String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-");
// assertThat(b).isEqualTo("-bval1-");
// }
//
@Test
public void shouldExpandNestedTemplates() {
StrInterpolator interp = new StrInterpolator(abcd);
String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
assertThat(a).isEqualTo("-aval1-");
String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-");
assertThat(b).isEqualTo("-bval1-");
}
// @Test
// public void shouldGetBasicDetails() {
// StrInterpolator interp = new StrInterpolator(abcd);
// LinkedHashMap<String, String> details = interp.getTemplateDetails("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
// assertThat(details).containsOnlyKeys("akey","dkey");
// assertThat(details).containsValues("test1");
//
// }
//
}

50
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorOutput.java Normal file
View File

@ -0,0 +1,50 @@
package io.nosqlbench.nb.api.advisor;
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Level;
public class NBAdvisorOutput {
private final static Logger logger = LogManager.getLogger("ADVISOR");
public static void render(Level level,String message) {
if (NBAdvisorLevel.get() == NBAdvisorLevel.validate) {
output(level, message);
}
}
public static void test(String message) {
output(Level.WARN, message);
if (NBAdvisorLevel.get() == NBAdvisorLevel.enforce) {
throw new NBAdvisorException(message, 2);
}
}
public static void output(Level level,String message) {
if (level == Level.INFO) {
logger.info(message);
} else if (level == Level.WARN) {
logger.warn(message);
} else if (level == Level.ERROR) {
logger.error(message);
}
}
}

1
nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml
View File

@ -6,5 +6,6 @@ scenarios:
bindings:
cycle: Identity()
name: NumberNameToCycle()
rw_key: TEMPLATE(keydist,Uniform(0,1000000000)->int); ToString() -> String
ops:
cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n"

69
nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java
View File

@ -312,10 +312,6 @@ public class NBCLIScenarioPreprocessor {
return parsedStep;
}
private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static List<WorkloadDesc> filterForScenarios(List<Content<?>> candidates) {
List<Path> yamlPathList = candidates.stream().map(Content::asPath).toList();
@ -437,28 +433,43 @@ public class NBCLIScenarioPreprocessor {
}
// private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
// private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static Map<String, String> matchTemplates(String line, Map<String, String> templates) {
Matcher matcher = templatePattern.matcher(line);
while (matcher.find()) {
String match = matcher.group(1);
Matcher innerMatcher = innerTemplatePattern.matcher(match);
if (innerMatcher.find()) {
String innerMatch = innerMatcher.group(1);
templates = matchTemplates("TEMPLATE(" + innerMatch + ")", templates);
String resolvedInner = templates.getOrDefault(innerMatch.split("[,:]")[0], "");
match = match.replace("TEMPLATE(" + innerMatch + ")", resolvedInner);
int length = line.length();
int i = 0;
while (i < length) {
// Detect an instance of "TEMPLATE("
if (line.startsWith("TEMPLATE(", i)) {
int start = i + "TEMPLATE(".length();
int openParensCount = 1; // We found one '(' with "TEMPLATE("
// Find the corresponding closing ')' for this TEMPLATE instance
int j = start;
while (j < length && openParensCount > 0) {
if (line.charAt(j) == '(') {
openParensCount++;
} else if (line.charAt(j) == ')') {
openParensCount--;
}
j++;
}
// `j` now points just after the closing ')' of this TEMPLATE
if (openParensCount == 0) {
String templateContent = line.substring(start, j - 1);
// Resolve the template content
String resolvedContent = resolveTemplate(templateContent, templates);
line = line.substring(0, i) + resolvedContent + line.substring(j);
// Update `length` and `i` based on the modified `line`
i += resolvedContent.length() - 1;
length = line.length();
}
}
i++;
}
String[] matchArray = match.split("[,:]");
if (matchArray.length == 1) {
matchArray = new String[]{matchArray[0], ""};
}
templates.put(matchArray[0], matchArray[1]);
}
matcher = templatePattern2.matcher(line);
Matcher matcher = templatePattern2.matcher(line);
while (matcher.find()) {
String match = matcher.group(1);
String[] matchArray = match.split(":");
@ -471,5 +482,17 @@ public class NBCLIScenarioPreprocessor {
return templates;
}
private static String resolveTemplate(String content, Map<String, String> templates) {
String[] parts = content.split("[,:]", 2);
String key = parts[0];
String value = parts.length > 1 ? parts[1] : "";
// Store or retrieve the resolved value for the template key
if (!templates.containsKey(key)) {
templates.put(key, value);
}
return templates.getOrDefault(key, "");
}
}