IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-01-19 12:13:05 -06:00
Code Issues Packages Projects Releases Wiki Activity

allow type-safe replacement of stmt semantics

Browse Source
This commit is contained in:
Jonathan Shook 2021-06-23 11:42:49 -05:00
parent e6e398bb6b
commit 5a1943e321
11 changed files with 42 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -28,7 +28,7 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
@ -182,7 +182,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
for (OpTemplate stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError();
ParsedStmtOp parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orElseThrow();
boolean prepared = stmtDef.getParamOrDefault("prepared", true);
boolean parameterized = stmtDef.getParamOrDefault("parameterized", false);
long ratio = stmtDef.getParamOrDefault("ratio", 1);
@ -345,7 +345,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
psummary.append(" logresultcsv=>").append(logresultcsv);
}
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints());
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().orElseThrow().getBindPoints());
if (psummary.length() > 0) {
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary);
@ -391,14 +391,14 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + " with compatibility mode. " +
"This will be deprecated in a future release.");
logger.warn("DEPRECATED-FORMAT: Please refer to " +
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
"http://docs.nosqlbench.io/ for more details.");
} else {
throw new BasicError("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + " with compatibility mode. " +
"This has been deprecated for a long time now. You should use the modern yaml format, which is easy" +
"to convert to. If you want to ignore this and kick the issue" +
" down the road to someone else, then you can add ignore_important_warnings=true. " +
"Please refer to " +
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
"http://docs.nosqlbench.io/ for more details.");
}
break;
case "2":
@ -415,7 +415,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc +
" with compatibility mode. This will be deprecated in a future release.");
logger.warn("DEPRECATED-FORMAT: Please refer to " +
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
"http://docs.nosqlbench.io/ for more details.");
} catch (Exception compatError) {
logger.warn("Tried to load yaml in compatibility mode, " +
"since it failed to load with the standard format, " +

8
driver-cqld3-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -28,7 +28,7 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
@ -179,9 +179,11 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
Set<String> timerStarts = new HashSet<>();
Set<String> timerStops = new HashSet<>();
Pattern questionPattern = Pattern.compile("\\?(?<arg>\\w+)");
for (OpTemplate stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError();
ParsedStmtOp parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orElseThrow();
boolean prepared = stmtDef.getParamOrDefault("prepared", true);
boolean parameterized = stmtDef.getParamOrDefault("parameterized", false);
long ratio = stmtDef.getParamOrDefault("ratio", 1);
@ -344,7 +346,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
psummary.append(" logresultcsv=>").append(logresultcsv);
}
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().getBindPoints());
template.getContextualBindings().getBindingsTemplate().addFieldBindings(stmtDef.getParsed().orElseThrow().getBindPoints());
if (psummary.length() > 0) {
logger.info("statement named '" + stmtDef.getName() + "' has custom settings:" + psummary);

16
driver-dsegraph-shaded/src/main/java/com/datastax/ebdrivers/dsegraph/GraphActivity.java
View File

@ -14,7 +14,7 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
@ -102,7 +102,7 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
for (OpTemplate stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError();
ParsedStmtOp parsed = stmtDef.getParsed().orElseThrow();
ReadyGraphStatementTemplate readyGraphStatement;
long ratio = Long.valueOf(stmtDef.getParams().getOrDefault("ratio", "1").toString());
@ -112,16 +112,16 @@ public class GraphActivity extends SimpleActivity implements ActivityDefObserver
if (repeat.isPresent()) {
readyGraphStatement = new ReadyGraphStatementTemplate(
stmtDef.getName(),
GraphStmtParser.getCookedRepeatedStatement(stmtDef.getStmt(), repeat.get()),
stmtDef.getParsed().getBindPoints(),
GraphStmtParser.getFields(stmtDef.getStmt(), stmtDef.getBindings()).toArray(new String[0]),
GraphStmtParser.getCookedRepeatedStatement(stmtDef.getStmt().orElseThrow(), repeat.get()),
stmtDef.getParsed().orElseThrow().getBindPoints(),
GraphStmtParser.getFields(stmtDef.getStmt().orElseThrow(), stmtDef.getBindings()).toArray(new String[0]),
repeat.get());
} else {
readyGraphStatement = new ReadyGraphStatementTemplate(
stmtDef.getName(),
GraphStmtParser.getCookedStatement(stmtDef.getStmt()),
stmtDef.getParsed().getBindPoints(),
GraphStmtParser.getFields(stmtDef.getStmt(), stmtDef.getBindings()).toArray(new String[0]));
GraphStmtParser.getCookedStatement(stmtDef.getStmt().orElseThrow()),
stmtDef.getParsed().orElseThrow().getBindPoints(),
GraphStmtParser.getFields(stmtDef.getStmt().orElseThrow(), stmtDef.getBindings()).toArray(new String[0]));
}
planner.addOp(readyGraphStatement, ratio);
}

4
driver-jdbc/src/main/java/io/nosqlbench/activitytype/jdbc/impl/ReadyJDBCOp.java
View File

@ -11,9 +11,9 @@ public class ReadyJDBCOp implements OpDispenser<String> {
private final StringBindings bindings;
public ReadyJDBCOp(OpTemplate stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt().orElseThrow(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt().orElseThrow(), paramBindings);
bindings = template.resolve();
}

4
driver-kafka/src/main/java/com/datastax/ebdrivers/kafkaproducer/KafkaStatement.java
View File

@ -32,9 +32,9 @@ public class KafkaStatement {
private final String key;
public KafkaStatement(OpTemplate stmtDef, String servers, String clientId, String schemaRegistryUrl) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt().orElseThrow(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt().orElseThrow(), paramBindings);
this.bindings = template.resolve();

4
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@ -11,7 +11,7 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
@ -117,7 +117,7 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
logger.error("No statements found for this activity");
} else {
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
ParsedStmtOp parsed = stmt.getParsed().orElseThrow();
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);

4
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java
View File

@ -15,9 +15,9 @@ public class ReadyMongoStatement {
private final ReadPreference readPreference;
public ReadyMongoStatement(OpTemplate stmtDef) {
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt(), stmtDef.getBindings());
ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt().orElseThrow(), stmtDef.getBindings());
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt().orElseThrow(), paramBindings);
this.bindings = template.resolve();
this.readPreference = stmtDef.getOptionalStringParam("readPreference")

8
driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java
View File

@ -1,6 +1,6 @@
package io.nosqlbench.driver.mongodb;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
@ -43,7 +43,7 @@ public class ReadyMongoStatementTest {
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
assertThat(stmts).hasSize(1);
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
ParsedStmtOp parsed = stmt.getParsed().orElseThrow();
assertThat(parsed.getBindPoints()).hasSize(2);
BindPoint seqKey = new BindPoint("seq_key", "Mod(1000000L); ToInt()");
@ -66,7 +66,7 @@ public class ReadyMongoStatementTest {
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
assertThat(stmts).hasSize(1);
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
ParsedStmtOp parsed = stmt.getParsed().orElseThrow();
assertThat(parsed.getBindPoints()).hasSize(1);
BindPoint rwKey = new BindPoint("rw_key", "Uniform(0,1000000)->long; ToInt()");
@ -88,7 +88,7 @@ public class ReadyMongoStatementTest {
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
assertThat(stmts).hasSize(1);
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
ParsedStmtOp parsed = stmt.getParsed().orElseThrow();
assertThat(parsed.getBindPoints()).hasSize(2);
BindPoint rwKey = new BindPoint("rw_key", "Uniform(0,1000000)->long; ToInt()");

6
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -23,7 +23,7 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.ParsedStmtOp;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
@ -180,7 +180,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
}
} else if (stmts.size() > 0) {
for (OpTemplate stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();
ParsedStmtOp parsed = stmt.getParsed().orElseThrow();
BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);
@ -188,7 +188,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
statement = statement + "\n";
}
StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt(), bt);
StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt().orElseThrow(), bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb, stmt.getParamOrDefault("ratio", 1));
}

6
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java
View File

@ -54,19 +54,19 @@ public class StmtEscapingTest {
@Test
public void testBackslashEscape() {
String s1 = defs.get(0).getStmt();
String s1 = defs.get(0).getStmt().orElseThrow();
assertThat(s1).isEqualTo("This is a \"statement\"");
}
@Test
public void testBackslashInBlock() {
String s2 = defs.get(1).getStmt();
String s2 = defs.get(1).getStmt().orElseThrow();
assertThat(s2).isEqualTo("This is a \\\"statement\\\".\n");
}
@Test
public void testTripleQuotesInBlock() {
String s3 = defs.get(2).getStmt();
String s3 = defs.get(2).getStmt().orElseThrow();
assertThat(s3).isEqualTo("This is a \"\"\"statement\"\"\".\n");
}

10
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java
View File

@ -44,7 +44,7 @@ public class StmtDetailOverrideTest {
assertThat(doc1block0.getOps().size()).isEqualTo(1);
OpTemplate s = doc1block0.getOps().get(0);
assertThat(s.getName()).isEqualTo("block0--stmt1");
assertThat(s.getStmt()).isEqualTo("globalstatement1");
assertThat(s.getStmt()).contains("globalstatement1");
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
assertThat(s.getTags()).hasSize(1);
@ -55,21 +55,21 @@ public class StmtDetailOverrideTest {
s = stmts.get(0);
assertThat(s.getName()).isEqualTo("testblock1--stmt1");
assertThat(s.getStmt()).isEqualTo("astatement1");
assertThat(s.getStmt()).contains("astatement1");
assertThat(s.getTags()).hasSize(1);
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
s = stmts.get(1);
assertThat(s.getName()).isEqualTo("testblock1--s2name");
assertThat(s.getStmt()).isEqualTo("s2statement data");
assertThat(s.getStmt()).contains("s2statement data");
assertThat(s.getTags()).hasSize(1);
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
s = stmts.get(2);
assertThat(s.getName()).isEqualTo("testblock1--s3");
assertThat(s.getStmt()).isEqualTo("statement three");
assertThat(s.getStmt()).contains("statement three");
assertThat(s.getTags()).containsEntry("tname1", "tval1");
assertThat(s.getTags()).containsEntry("global_tag1", "tag value");
assertThat(s.getBindings()).hasSize(3);
@ -77,7 +77,7 @@ public class StmtDetailOverrideTest {
s = stmts.get(3);
assertThat(s.getName()).isEqualTo("testblock1--s4");
assertThat(s.getStmt()).isEqualTo("statement 4");
assertThat(s.getStmt()).contains("statement 4");
assertThat(s.getTags()).hasSize(1);
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);