IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-01-13 01:12:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

#158 Simplify YamlStatement Loader

Browse Source
This commit is contained in:
Jonathan Shook 2020-06-11 12:21:44 -05:00
parent 941f08c017
commit 46b2447d81
27 changed files with 583 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java
View File

@ -137,7 +137,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private void initSequencer() {
Session session = getSession();
Map<String,Object> fconfig = Map.of("session",session);
Map<String, Object> fconfig = Map.of("session", session);
SequencerType sequencerType = SequencerType.valueOf(
getParams().getOptionalString("seq").orElse("bucket")
@ -160,33 +160,28 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
for (StmtDef stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError();
boolean prepared = Boolean.valueOf(stmtDef.getParams().getOrDefault("prepared", "true"));
boolean parametrized = Boolean.valueOf(stmtDef.getParams().getOrDefault("parametrized", "false"));
long ratio = Long.valueOf(stmtDef.getParams().getOrDefault("ratio", "1"));
boolean prepared = stmtDef.getParamOrDefault("prepared", true);
boolean parametrized = stmtDef.getParamOrDefault("parametrized", false);
long ratio = stmtDef.getParamOrDefault("ratio", 1);
Optional<ConsistencyLevel> cl = Optional.ofNullable(
stmtDef.getParams().getOrDefault("cl", null)).map(ConsistencyLevel::valueOf);
Optional<ConsistencyLevel> serial_cl = Optional.ofNullable(
stmtDef.getParams().getOrDefault("serial_cl", null)).map(ConsistencyLevel::valueOf);
Optional<Boolean> idempotent = Optional.ofNullable(stmtDef.getParams().getOrDefault("idempotent", null))
.map(Boolean::valueOf);
Optional<ConsistencyLevel> cl = stmtDef.getOptionalParam("cl", String.class).map(ConsistencyLevel::valueOf);
Optional<ConsistencyLevel> serial_cl = stmtDef.getOptionalParam("serial_cl").map(ConsistencyLevel::valueOf);
Optional<Boolean> idempotent = stmtDef.getOptionalParam("idempotent", boolean.class);
StringBuilder psummary = new StringBuilder();
boolean instrument = Optional.ofNullable(stmtDef.getParams()
.get("instrument")).map(Boolean::valueOf)
boolean instrument = stmtDef.getOptionalParam("instrument", boolean.class)
.orElse(getParams().getOptionalBoolean("instrument").orElse(false));
String logresultcsv = stmtDef.getParams().getOrDefault("logresultcsv","");
String logresultcsv = stmtDef.getParamOrDefault("logresultcsv", "");
String logresultcsv_act = getParams().getOptionalString("logresultcsv").orElse("");
if (!logresultcsv_act.isEmpty() && !logresultcsv_act.toLowerCase().equals("true")) {
throw new RuntimeException("At the activity level, only logresultcsv=true is allowed, no other values.");
}
logresultcsv = !logresultcsv.isEmpty() ? logresultcsv : logresultcsv_act;
logresultcsv = !logresultcsv.toLowerCase().equals("true") ? logresultcsv : stmtDef.getName()+"--results.csv";
logresultcsv = !logresultcsv.toLowerCase().equals("true") ? logresultcsv : stmtDef.getName() + "--results.csv";
logger.debug("readying statement[" + (prepared ? "" : "un") + "prepared]:" + parsed.getStmt());
@ -207,11 +202,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
psummary.append(" idempotent=").append(idempotent);
prepare.setIdempotent(i);
});
CqlBinderTypes binderType = CqlBinderTypes.valueOf(stmtDef.getParams()
.getOrDefault("binder", CqlBinderTypes.DEFAULT.toString()));
CqlBinderTypes binderType = stmtDef.getOptionalParam("binder")
.map(CqlBinderTypes::valueOf)
.orElse(CqlBinderTypes.DEFAULT);
template = new ReadyCQLStatementTemplate(fconfig, binderType, getSession(), prepare, ratio,
parsed.getName());
parsed.getName());
} else {
SimpleStatement simpleStatement = new SimpleStatement(stmtForDriver);
cl.ifPresent((conlvl) -> {
@ -227,18 +224,18 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
simpleStatement.setIdempotent(i);
});
template = new ReadyCQLStatementTemplate(fconfig, getSession(), simpleStatement, ratio,
parsed.getName(), parametrized);
parsed.getName(), parametrized);
}
Optional.ofNullable(stmtDef.getParams().getOrDefault("save", null))
.map(s -> s.split("[,; ]"))
stmtDef.getOptionalParam("save")
.map(s -> s.split("[,: ]"))
.map(Save::new)
.ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString());
template.addRowCycleOperators(save_op);
});
Optional.ofNullable(stmtDef.getParams().getOrDefault("rsoperators", null))
stmtDef.getOptionalParam("rsoperators")
.map(s -> s.split(","))
.stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator)
@ -247,7 +244,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
template.addResultSetOperators(rso);
});
Optional.ofNullable(stmtDef.getParams().getOrDefault("rowoperators", null))
stmtDef.getOptionalParam("rowoperators")
.map(s -> s.split(","))
.stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator)
@ -264,7 +261,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
if (!logresultcsv.isEmpty()) {
logger.info("Adding per-statement result CSV logging to statement '" + parsed.getName() + "'");
template.logResultCsv(this,logresultcsv);
template.logResultCsv(this, logresultcsv);
psummary.append(" logresultcsv=>").append(logresultcsv);
}
@ -357,7 +354,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
rawblock.setTags(rawdef.getTags());
// params
Map<String, String> params = new HashMap<>(rawdef.getParams());
Map<String, Object> params = new HashMap<>(rawdef.getParams());
if (rawstmt.getConsistencyLevel() != null && !rawstmt.getConsistencyLevel().isEmpty())
params.put("cl", rawstmt.getConsistencyLevel());
if (!rawstmt.isPrepared()) params.put("prepared", "false");
@ -516,7 +513,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
if (keyval.length == 1) {
String verb = keyval[0];
ErrorResponse errorResponse = getErrorResponseOrBasicError(verb);
newerrorHandler.setDefaultHandler(
newerrorHandler.setDefaultHandler(
new NBCycleErrorHandler(
errorResponse,
exceptionCountMetrics,
@ -558,8 +555,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
private ErrorResponse getErrorResponseOrBasicError(String verb) {
try {
return ErrorResponse.valueOf(verb);
}catch(IllegalArgumentException e){
throw new BasicError("Invalid parameter for errors: '"+ verb + "' should be one of: " + StringUtils.join(ErrorResponse.values(), ", "));
} catch (IllegalArgumentException e) {
throw new BasicError("Invalid parameter for errors: '" + verb + "' should be one of: " + StringUtils.join(ErrorResponse.values(), ", "));
}
}

213
driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/core/CqlActivity.java
View File

@ -143,7 +143,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
Map<String, Object> fconfig = Map.of("session", session);
SequencerType sequencerType = SequencerType.valueOf(
getParams().getOptionalString("seq").orElse("bucket")
getParams().getOptionalString("seq").orElse("bucket")
);
SequencePlanner<ReadyCQLStatement> planner = new SequencePlanner<>(sequencerType);
@ -163,17 +163,19 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
for (StmtDef stmtDef : stmts) {
ParsedStmt parsed = stmtDef.getParsed().orError();
boolean prepared = Boolean.valueOf(stmtDef.getParams().getOrDefault("prepared", "true"));
boolean parametrized = Boolean.valueOf(stmtDef.getParams().getOrDefault("parametrized", "false"));
long ratio = Long.valueOf(stmtDef.getParams().getOrDefault("ratio", "1"));
boolean prepared = stmtDef.getParamOrDefault("prepared", true);
boolean parametrized = stmtDef.getParamOrDefault("parametrized", false);
long ratio = stmtDef.getParamOrDefault("ratio", 1);
StringBuilder psummary = new StringBuilder();
boolean instrument = Optional.ofNullable(stmtDef.getParams()
.get("instrument")).map(Boolean::valueOf)
.orElse(getParams().getOptionalBoolean("instrument").orElse(false));
boolean instrument = stmtDef.getOptionalParam("instrument")
.or(() -> getParams().getOptionalString("instrument"))
.map(Boolean::parseBoolean)
.orElse(false);
String logresultcsv = stmtDef.getParams().getOrDefault("logresultcsv", "");
String logresultcsv = stmtDef.getParamOrDefault("logresultcsv","");
String logresultcsv_act = getParams().getOptionalString("logresultcsv").orElse("");
if (!logresultcsv_act.isEmpty() && !logresultcsv_act.toLowerCase().equals("true")) {
@ -190,77 +192,79 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
SimpleStatementBuilder stmtBuilder = SimpleStatement.builder(stmtForDriver);
psummary.append(" statement=>").append(stmtForDriver);
Optional.ofNullable(stmtDef.getParams().getOrDefault("cl", null))
.map(DefaultConsistencyLevel::valueOf)
.map(conlvl -> {
psummary.append(" consistency_level=>").append(conlvl);
return conlvl;
})
.ifPresent(stmtBuilder::setConsistencyLevel);
Optional.ofNullable(stmtDef.getParams().getOrDefault("serial_cl", null))
.map(DefaultConsistencyLevel::valueOf)
.map(sconlvel -> {
psummary.append(" serial_consistency_level=>").append(sconlvel);
return sconlvel;
})
.ifPresent(stmtBuilder::setSerialConsistencyLevel);
stmtDef.getOptionalParam("cl")
.map(DefaultConsistencyLevel::valueOf)
.map(conlvl -> {
psummary.append(" consistency_level=>").append(conlvl);
return conlvl;
})
.ifPresent(stmtBuilder::setConsistencyLevel);
Optional.ofNullable(stmtDef.getParams().getOrDefault("idempotent", null))
.map(Boolean::valueOf)
.map(idempotent -> {
psummary.append(" idempotent=").append(idempotent);
return idempotent;
})
.ifPresent(stmtBuilder::setIdempotence);
stmtDef.getOptionalParam("serial_cl")
.map(DefaultConsistencyLevel::valueOf)
.map(sconlvel -> {
psummary.append(" serial_consistency_level=>").append(sconlvel);
return sconlvel;
})
.ifPresent(stmtBuilder::setSerialConsistencyLevel);
stmtDef.getOptionalParam("idempotent")
.map(Boolean::valueOf)
.map(idempotent -> {
psummary.append(" idempotent=").append(idempotent);
return idempotent;
})
.ifPresent(stmtBuilder::setIdempotence);
if (prepared) {
PreparedStatement preparedStatement = getSession().prepare(stmtBuilder.build());
CqlBinderTypes binderType = CqlBinderTypes.valueOf(stmtDef.getParams()
.getOrDefault("binder", CqlBinderTypes.DEFAULT.toString()));
CqlBinderTypes binderType = stmtDef.getOptionalParam("binder")
.map(CqlBinderTypes::valueOf)
.orElse(CqlBinderTypes.DEFAULT);
template = new ReadyCQLStatementTemplate(
fconfig,
binderType,
getSession(),
preparedStatement,
ratio,
parsed.getName()
fconfig,
binderType,
getSession(),
preparedStatement,
ratio,
parsed.getName()
);
} else {
SimpleStatement simpleStatement = SimpleStatement.newInstance(stmtForDriver);
template = new ReadyCQLStatementTemplate(fconfig, getSession(), simpleStatement, ratio,
parsed.getName(), parametrized);
parsed.getName(), parametrized);
}
Optional.ofNullable(stmtDef.getParams().getOrDefault("save", null))
.map(s -> s.split("[,; ]"))
.map(Save::new)
.ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString());
template.addRowCycleOperators(save_op);
});
stmtDef.getOptionalParam("save")
.map(s -> s.split("[,; ]"))
.map(Save::new)
.ifPresent(save_op -> {
psummary.append(" save=>").append(save_op.toString());
template.addRowCycleOperators(save_op);
});
Optional.ofNullable(stmtDef.getParams().getOrDefault("rsoperators", null))
.map(s -> s.split(","))
.stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator)
.forEach(rso -> {
psummary.append(" rsop=>").append(rso.toString());
template.addResultSetOperators(rso);
});
stmtDef.getOptionalParam("rsoperators")
.map(s -> s.split(","))
.stream().flatMap(Arrays::stream)
.map(ResultSetCycleOperators::newOperator)
.forEach(rso -> {
psummary.append(" rsop=>").append(rso.toString());
template.addResultSetOperators(rso);
});
Optional.ofNullable(stmtDef.getParams().getOrDefault("rowoperators", null))
.map(s -> s.split(","))
.stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator)
.forEach(ro -> {
psummary.append(" rowop=>").append(ro.toString());
template.addRowCycleOperators(ro);
});
stmtDef.getOptionalParam("rowoperators")
.map(s -> s.split(","))
.stream().flatMap(Arrays::stream)
.map(RowCycleOperators::newOperator)
.forEach(ro -> {
psummary.append(" rowop=>").append(ro.toString());
template.addRowCycleOperators(ro);
});
if (instrument) {
logger.info("Adding per-statement success and error and resultset-size timers to statement '" + parsed.getName() + "'");
@ -305,9 +309,9 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
case "1":
doclist = getVersion1StmtsDoc(interp, yaml_loc);
logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + " with compatibility mode. " +
"This will be deprecated in a future release.");
"This will be deprecated in a future release.");
logger.warn("DEPRECATED-FORMAT: Please refer to " +
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
break;
case "2":
doclist = StatementsLoader.load(logger, yaml_loc, interp, "activities");
@ -315,22 +319,22 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
case "unset":
try {
logger.debug("You can suffix your yaml filename or url with the " +
"format version, such as :1 or :2. Assuming version 2.");
"format version, such as :1 or :2. Assuming version 2.");
doclist = StatementsLoader.load(null, yaml_loc, interp, "activities");
} catch (Exception ignored) {
try {
doclist = getVersion1StmtsDoc(interp, yaml_loc);
logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc +
" with compatibility mode. This will be deprecated in a future release.");
" with compatibility mode. This will be deprecated in a future release.");
logger.warn("DEPRECATED-FORMAT: Please refer to " +
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
"http://docs.engineblock.io/user-guide/standard_yaml/ for more details.");
} catch (Exception compatError) {
logger.warn("Tried to load yaml in compatibility mode, " +
"since it failed to load with the standard format, " +
"but found an error:" + compatError);
"since it failed to load with the standard format, " +
"but found an error:" + compatError);
logger.warn("The following detailed errors are provided only " +
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
"for the standard format. To force loading version 1 with detailed logging, add" +
" a version qualifier to your yaml filename or url like ':1'");
// retrigger the error again, this time with logging enabled.
doclist = StatementsLoader.load(logger, yaml_loc, interp, "activities");
}
@ -338,7 +342,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
break;
default:
throw new RuntimeException("Unrecognized yaml format version, expected :1 or :2 " +
"at end of yaml file, but got " + yamlVersion + " instead.");
"at end of yaml file, but got " + yamlVersion + " instead.");
}
return doclist;
@ -363,12 +367,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
rawblock.setTags(rawdef.getTags());
// params
Map<String, String> params = new HashMap<>(rawdef.getParams());
Map<String, Object> params = new HashMap<>(rawdef.getParams());
if (rawstmt.getConsistencyLevel() != null && !rawstmt.getConsistencyLevel().isEmpty())
params.put("cl", rawstmt.getConsistencyLevel());
if (!rawstmt.isPrepared()) params.put("prepared", "false");
if (rawstmt.getRatio() != 1L)
params.put("ratio", String.valueOf(rawstmt.getRatio()));
rawblock.setParams(params);
@ -401,10 +406,10 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
@Override
public String toString() {
return "CQLActivity {" +
"activityDef=" + activityDef +
", session=" + session +
", opSequence=" + this.opsequence +
'}';
"activityDef=" + activityDef +
", session=" + session +
", opSequence=" + this.opsequence +
'}';
}
@Override
@ -420,7 +425,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
if (fetchSizeOption.isPresent()) {
int fetchSize = fetchSizeOption.flatMap(Unit::bytesFor).map(Double::intValue).orElseThrow(() -> new RuntimeException(
"Unable to parse fetch size from " + fetchSizeOption.get()
"Unable to parse fetch size from " + fetchSizeOption.get()
));
if (fetchSize > 10000000 && fetchSize < 1000000000) {
logger.warn("Setting the fetchsize to " + fetchSize + " is unlikely to give good performance.");
@ -429,7 +434,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
}
logger.trace("setting fetchSize to " + fetchSize);
CQLSessionCache.get().getSession(activityDef).set(DefaultDriverOption.REQUEST_PAGE_SIZE,fetchSize);
CQLSessionCache.get().getSession(activityDef).set(DefaultDriverOption.REQUEST_PAGE_SIZE, fetchSize);
}
this.retryDelay = params.getOptionalLong("retrydelay").orElse(0L);
@ -440,8 +445,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
this.maxpages = params.getOptionalInteger("maxpages").orElse(1);
this.statementFilter = params.getOptionalString("tokens")
.map(s -> new TokenRangeStmtFilter(getSession(), s))
.orElse(null);
.map(s -> new TokenRangeStmtFilter(getSession(), s))
.orElse(null);
if (statementFilter != null) {
logger.info("filtering statements" + statementFilter);
@ -450,13 +455,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
errorHandler = configureErrorHandler();
params.getOptionalString("trace")
.map(SimpleConfig::new)
.map(TraceLogger::new)
.ifPresent(
tl -> {
addResultSetCycleOperator(tl);
addStatementModifier(tl);
});
.map(SimpleConfig::new)
.map(TraceLogger::new)
.ifPresent(
tl -> {
addResultSetCycleOperator(tl);
addStatementModifier(tl);
});
this.maxTotalOpsInFlight = params.getOptionalLong("async").orElse(1L);
@ -514,8 +519,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
HashedCQLErrorHandler newerrorHandler = new HashedCQLErrorHandler(exceptionCountMetrics);
String errors = activityDef.getParams()
.getOptionalString("errors")
.orElse("stop,retryable->retry,unverified->stop");
.getOptionalString("errors")
.orElse("stop,retryable->retry,unverified->stop");
String[] handlerSpecs = errors.split(",");
@ -524,31 +529,31 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef
if (keyval.length == 1) {
String verb = keyval[0];
newerrorHandler.setDefaultHandler(
new NBCycleErrorHandler(
ErrorResponse.valueOf(verb),
exceptionCountMetrics,
exceptionHistoMetrics,
getParams().getOptionalLong("async").isEmpty()
)
new NBCycleErrorHandler(
ErrorResponse.valueOf(verb),
exceptionCountMetrics,
exceptionHistoMetrics,
getParams().getOptionalLong("async").isEmpty()
)
);
} else {
String pattern = keyval[0];
String verb = keyval[1];
if (newerrorHandler.getGroupNames().contains(pattern)) {
NBCycleErrorHandler handler =
new NBCycleErrorHandler(
ErrorResponse.valueOf(verb),
exceptionCountMetrics,
exceptionHistoMetrics,
getParams().getOptionalLong("async").isEmpty()
);
new NBCycleErrorHandler(
ErrorResponse.valueOf(verb),
exceptionCountMetrics,
exceptionHistoMetrics,
getParams().getOptionalLong("async").isEmpty()
);
logger.info("Handling error group '" + pattern + "' with handler:" + handler);
newerrorHandler.setHandlerForGroup(pattern, handler);
} else {
NBCycleErrorHandler handler = new NBCycleErrorHandler(
ErrorResponse.valueOf(keyval[1]),
exceptionCountMetrics,
exceptionHistoMetrics,
ErrorResponse.valueOf(keyval[1]),
exceptionCountMetrics,
exceptionHistoMetrics,
getParams().getOptionalLong("async").isEmpty()
);
logger.info("Handling error pattern '" + pattern + "' with handler:" + handler);

78
driver-cqld4/src/test/java/com/datastax/ebdrivers/cql/statements/CQLOptionsTest.java
View File

@ -1,78 +0,0 @@
package com.datastax.ebdrivers.cql.statements;
import com.datastax.driver.core.HostDistance;
import com.datastax.driver.core.PoolingOptions;
import com.datastax.driver.core.SocketOptions;
import com.datastax.driver.core.policies.LoadBalancingPolicy;
import com.datastax.driver.core.policies.ReconnectionPolicy;
import com.datastax.driver.core.policies.SpeculativeExecutionPolicy;
import io.nosqlbench.activitytype.cqld4.core.CQLOptions;
import org.junit.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class CQLOptionsTest {
@Test
public void testSpeculative() {
SpeculativeExecutionPolicy p1 = CQLOptions.speculativeFor("p99:5");
assertThat(p1).isNotNull();
SpeculativeExecutionPolicy p2 = CQLOptions.speculativeFor("p99:5:5000ms");
assertThat(p2).isNotNull();
}
@Test
public void testConstant() {
SpeculativeExecutionPolicy p1 = CQLOptions.speculativeFor("5000ms:5");
assertThat(p1).isNotNull();
}
@Test
public void testWhitelist() {
LoadBalancingPolicy lbp = CQLOptions.whitelistFor("127.0.0.1,127.0.0.2:123", null);
assertThat(lbp).isNotNull();
}
@Test
public void testReconnectPolicyPatterns() {
ReconnectionPolicy rp = CQLOptions.reconnectPolicyFor("exponential(123,321)");
rp = CQLOptions.reconnectPolicyFor("constant(123)");
}
@Test
public void testSocketOptionPatterns() {
SocketOptions so = CQLOptions.socketOptionsFor("read_timeout_ms=23423,connect_timeout_ms=2344;keep_alive:true,reuse_address:true;so_linger:323;tcp_no_delay=true;receive_buffer_size:100,send_buffer_size=1000");
assertThat(so.getConnectTimeoutMillis()).isEqualTo(2344);
assertThat(so.getKeepAlive()).isEqualTo(true);
assertThat(so.getReadTimeoutMillis()).isEqualTo(23423);
assertThat(so.getReceiveBufferSize()).isEqualTo(100);
assertThat(so.getReuseAddress()).isEqualTo(true);
assertThat(so.getSendBufferSize()).isEqualTo(1000);
assertThat(so.getSoLinger()).isEqualTo(323);
assertThat(so.getTcpNoDelay()).isEqualTo(true);
}
@Test
public void testConnectionsPatterns() {
PoolingOptions po = CQLOptions.poolingOptionsFor("2345");
assertThat(po.getCoreConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(2345);
assertThat(po.getMaxConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(Integer.MIN_VALUE);
assertThat(po.getMaxRequestsPerConnection(HostDistance.LOCAL)).isEqualTo(Integer.MIN_VALUE);
PoolingOptions po2 = CQLOptions.poolingOptionsFor("1:2:3,4:5:6");
assertThat(po2.getCoreConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(1);
assertThat(po2.getMaxConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(2);
assertThat(po2.getMaxRequestsPerConnection(HostDistance.LOCAL)).isEqualTo(3);
assertThat(po2.getCoreConnectionsPerHost(HostDistance.REMOTE)).isEqualTo(4);
assertThat(po2.getMaxConnectionsPerHost(HostDistance.REMOTE)).isEqualTo(5);
assertThat(po2.getMaxRequestsPerConnection(HostDistance.REMOTE)).isEqualTo(6);
PoolingOptions po3 = CQLOptions.poolingOptionsFor("1:2:3,4:5:6,heartbeat_interval_s:100,idle_timeout_s:123,pool_timeout_ms:234");
assertThat(po3.getIdleTimeoutSeconds()).isEqualTo(123);
assertThat(po3.getPoolTimeoutMillis()).isEqualTo(234);
assertThat(po3.getHeartbeatIntervalSeconds()).isEqualTo(100);
}
}

4
driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java
View File

@ -55,8 +55,8 @@ public class CqlVerifyActivity extends CqlActivity {
Map<String, String> bindings = stmts.get(0).getBindings();
if (stmts.get(0).getParams().containsKey("verify-fields")) {
List<String> fields = new ArrayList<>();
String fieldsSpec = stmts.get(0).getParams().get("verify-fields");
String[] vfields = fieldsSpec.split("\\s*,\\s*");
String fieldSpec= stmts.get(0).getParamOrDefault("verify-fields","");
String[] vfields = fieldSpec.split("\\s*,\\s*");
for (String vfield : vfields) {
if (vfield.equals("*")) {
bindings.forEach((k,v)->fields.add(k));

2
driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java
View File

@ -109,7 +109,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe
StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt(), bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb,Long.valueOf(stmt.getParams().getOrDefault("ratio","1")));
sequencer.addOp(sb,stmt.getParamOrDefault("ratio",1));
}
} else {
logger.error("Unable to create an HTTP statement if no bindings or statements are defined.");

3
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java
View File

@ -114,8 +114,7 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);
sequencer.addOp(new ReadyMongoStatement(stmt),
Long.parseLong(stmt.getParams().getOrDefault("ratio","1")));
sequencer.addOp(new ReadyMongoStatement(stmt), stmt.getParamOrDefault("ratio",1));
}
return sequencer.resolve();

5
driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java
View File

@ -20,8 +20,9 @@ public class ReadyMongoStatement {
StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings);
this.bindings = template.resolve();
this.readPreference = ReadPreference.valueOf(stmtDef.getParams()
.getOrDefault("readPreference","primary"));
this.readPreference = stmtDef.getOptionalParam("readPreference")
.map(ReadPreference::valueOf)
.orElse(ReadPreference.primary());
}
public ReadPreference getReadPreference() {

2
driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -167,7 +167,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt(), bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb,Long.parseLong(stmt.getParams().getOrDefault("ratio","1")));
sequencer.addOp(sb,stmt.getParamOrDefault("ratio",1));
}
} else {
logger.error("Unable to create a stdout statement if you have no active statements or bindings configured.");

2
driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java
View File

@ -30,7 +30,7 @@ public class CommandTemplate {
this.name = stmt.getName();
String prefixed = "command=" + stmt.getStmt();
Map<String,String> cmdMap = ParamsParser.parse(prefixed, canonicalize);
Map<String, String> paramsMap = stmt.getParams();
Map<String, String> paramsMap = stmt.getParamsAsValueType(String.class);
paramsMap.forEach((k,v) -> {
if (cmdMap.containsKey(k)) {
logger.warn("command property override: '" + k + "' superseded by param form with value '" + v + "'");

2
driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java
View File

@ -184,7 +184,7 @@ public class WebDriverActivity extends SimpleActivity {
}
for (StmtDef optemplate : stmts) {
long ratio = Long.parseLong(optemplate.getParams().getOrDefault("ratio", "1"));
long ratio = optemplate.getParamOrDefault("ratio", 1);
CommandTemplate cmd = new CommandTemplate(optemplate, false);
planner.addOp(cmd, ratio);
}

32
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/MultiMapLookup.java
View File

@ -17,22 +17,24 @@
package io.nosqlbench.engine.api.activityconfig;
import org.jetbrains.annotations.NotNull;
import java.util.*;
import java.util.stream.Collectors;
public class MultiMapLookup implements Map<String, String> {
public class MultiMapLookup<V> implements Map<String, V> {
private final List<Map<String, String>> maps = new ArrayList<>();
private final List<Map<String, V>> maps = new ArrayList<>();
public MultiMapLookup() {
}
public MultiMapLookup(Map<String,String> map1, Map<String,String> map2) {
public MultiMapLookup(Map<String,V> map1, Map<String,V> map2) {
add(map1);
add(map2);
}
public MultiMapLookup add(Map<String,String> map) {
public MultiMapLookup<V> add(Map<String,V> map) {
maps.add(map);
return this;
}
@ -59,26 +61,26 @@ public class MultiMapLookup implements Map<String, String> {
}
@Override
public String get(Object key) {
public V get(Object key) {
return maps.stream()
.filter(m -> m.containsKey(String.valueOf(key)))
.findFirst()
.map(m -> String.valueOf(m.get(key)))
.map(m -> m.get(key))
.orElse(null);
}
@Override
public String put(String key, String value) {
public V put(String key, V value) {
throw immutable();
}
@Override
public String remove(Object key) {
public V remove(Object key) {
throw immutable();
}
@Override
public void putAll(Map<? extends String, ? extends String> m) {
public void putAll(Map<? extends String, ? extends V> m) {
throw immutable();
}
@ -88,6 +90,7 @@ public class MultiMapLookup implements Map<String, String> {
}
@Override
@NotNull
public Set<String> keySet() {
Set<String> keys = new HashSet<>();
maps.stream().map(Map::keySet).flatMap(Set::stream)
@ -96,18 +99,19 @@ public class MultiMapLookup implements Map<String, String> {
}
@Override
public Collection<String> values() {
@NotNull
public Collection<V> values() {
return entrySet().stream()
.map(Entry::getValue)
.collect(Collectors.toList());
}
@Override
public Set<Entry<String, String>> entrySet() {
Map<String, String> compositeMap = new HashMap<>();
public Set<Entry<String, V>> entrySet() {
Map<String, V> compositeMap = new HashMap<>();
for (Map<String, String> map : maps) {
for (Entry<String, String> entry : map.entrySet()) {
for (Map<String, V> map : maps) {
for (Entry<String, V> entry : map.entrySet()) {
if (!compositeMap.containsKey(entry.getKey())) {
compositeMap.put(entry.getKey(), entry.getValue());
}

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java
View File

@ -153,7 +153,7 @@ public class ParsedStmt {
/**
* @return the params from the enclosed {@link StmtDef}
*/
public Map<String, String> getParams() {
public Map<String, Object> getParams() {
return stmtDef.getParams();
}

47
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -17,8 +17,10 @@
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlStatementLoader;
import io.nosqlbench.engine.api.activityconfig.stmtloader.GenericStmtLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
@ -28,6 +30,11 @@ import java.util.function.Function;
public class StatementsLoader {
public enum Loader {
original,
generified
}
public static StmtsDocList load(Logger logger, Content<?> content) {
RawYamlStatementLoader loader = new RawYamlStatementLoader();
RawStmtsDocList rawDocList = loader.loadString(logger, content.get());
@ -42,18 +49,32 @@ public class StatementsLoader {
// return layered;
// }
public static StmtsDocList load(Loader loader, Logger logger, String path, String... searchPaths) {
RawStmtsDocList list= null;
switch (loader) {
case generified:
GenericStmtLoader gloaderImpl = new GenericStmtLoader();
list = gloaderImpl.load(logger,path,searchPaths);
case original:
RawYamlStatementLoader yloaderImpl = new RawYamlStatementLoader();
list = yloaderImpl.load(logger, path, searchPaths);
}
return new StmtsDocList(list);
}
public static StmtsDocList load(Logger logger, String path, String... searchPaths) {
Content<?> content = NBIO.all()
.prefix(searchPaths)
.name(path)
.one();
.prefix(searchPaths)
.name(path)
.one();
try {
RawYamlStatementLoader loader = new RawYamlStatementLoader();
RawStmtsDocList rawDocList = loader.loadString(logger, content.get());
StmtsDocList layered = new StmtsDocList(rawDocList);
return layered;
} catch (Exception e) {
throw new RuntimeException("error while reading file " + path,e);
throw new RuntimeException("error while reading file " + path, e);
}
}
@ -64,4 +85,22 @@ public class StatementsLoader {
return layered;
}
public static StmtsDocList load(Loader loader, Logger logger, String path,
Function<String,String> transformer, String... searchPaths) {
RawStmtsDocList list= null;
switch (loader) {
case generified:
GenericStmtLoader gloaderImpl = new GenericStmtLoader();
gloaderImpl.addTransformers(transformer);
list = gloaderImpl.load(logger,path,searchPaths);
case original:
RawYamlStatementLoader yloaderImpl = new RawYamlStatementLoader();
yloaderImpl.addTransformer(transformer);
list = yloaderImpl.load(logger, path, searchPaths);
}
return new StmtsDocList(list);
}
}

59
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BlockParams.java
View File

@ -18,6 +18,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@ -25,10 +26,11 @@ public class BlockParams extends Tags {
private String name = "";
private String desc = "";
private Map<String, String> bindings = new LinkedHashMap<>();
private Map<String, String> params = new LinkedHashMap<>();
private final Map<String, String> bindings = new LinkedHashMap<>();
private final Map<String, Object> params = new LinkedHashMap<>();
public BlockParams() {}
public BlockParams() {
}
public String getDesc() {
return desc;
@ -37,6 +39,7 @@ public class BlockParams extends Tags {
public void setDesc(String desc) {
this.desc = desc;
}
public void setDescription(String desc) {
this.desc = desc;
}
@ -58,11 +61,17 @@ public class BlockParams extends Tags {
this.bindings.putAll(bindings);
}
public Map<String, String> getParams() {
return params;
public Map<String,Object> getParams() {
return this.params;
}
public void setParams(Map<String, String> config) {
// public Map<String, String> getParamsAsText() {
// Map<String,String> paramsMap = new HashMap<>();
// this.params.forEach((ko,vo) -> paramsMap.put(ko,vo.toString()));
// return paramsMap;
// }
public void setParams(Map<String, Object> config) {
this.params.clear();
this.params.putAll(config);
}
@ -73,4 +82,40 @@ public class BlockParams extends Tags {
setTags(other.getTags());
setParams(other.getParams());
}
}
@SuppressWarnings("unchecked")
public void setFieldsByReflection(Map<String, Object> propsmap) {
Object descriptionObj = propsmap.remove("description");
if (descriptionObj!=null) {
setDescription(descriptionObj.toString());
}
Object nameObj = propsmap.remove("name");
if (nameObj!=null) {
setName(nameObj.toString());
}
Object bindingsObject = propsmap.remove("bindings");
if (bindingsObject!=null) {
if (bindingsObject instanceof Map) {
Map<Object,Object> bindingsMap = (Map<Object,Object>) bindingsObject;
bindingsMap.forEach((ko,vo) -> bindings.put(ko.toString(), vo.toString()));
} else {
throw new RuntimeException("Invalid type for bindings object: " + bindingsObject.getClass().getCanonicalName());
}
}
Object paramsObject = propsmap.remove("params");
if (paramsObject!=null) {
if (paramsObject instanceof Map) {
Map<Object,Object> paramsMap = (Map<Object,Object>) paramsObject;
paramsMap.forEach((ko,vo) -> params.put(ko.toString(),vo));
} else {
throw new RuntimeException("Invalid type for params object:" + paramsObject.getClass().getCanonicalName());
}
}
super.setFieldsByReflection(propsmap);
}
}

6
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawScenarios.java
View File

@ -2,7 +2,7 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.*;
public class RawScenarios extends LinkedHashMap<String, LinkedList<String>> {
public class RawScenarios extends LinkedHashMap<String, Object> {
public static String STEPNAME = "%03d";
@ -10,6 +10,10 @@ public class RawScenarios extends LinkedHashMap<String, LinkedList<String>> {
return new LinkedList<>(this.keySet());
}
public void setPropertiesByReflection(Map<String,Object> scenarioInfo) {
scenarioInfo.forEach(this::put);
}
public Map<String,String> getNamedScenario(String scenarioName) {
Object v = this.get(scenarioName);

9
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java
View File

@ -33,6 +33,7 @@ public class RawStmtDef extends BlockParams {
this.statement = statement;
}
@SuppressWarnings("unchecked")
public RawStmtDef(String defaultName, Map<String, Object> map) {
@ -44,7 +45,7 @@ public class RawStmtDef extends BlockParams {
Optional.ofNullable((Map<String, String>) map.remove("tags")).ifPresent(this::setTags);
Optional.ofNullable((Map<String, String>) map.remove("bindings")).ifPresent(this::setBindings);
Optional.ofNullable((Map<String, String>) map.remove("params")).ifPresent(this::setParams);
Optional.ofNullable((Map<String, Object>) map.remove("params")).ifPresent(this::setParams);
// Depends on order stability, relying on LinkedHashMap -- Needs stability unit tests
@ -85,6 +86,10 @@ public class RawStmtDef extends BlockParams {
}
public String getName() {
return getParams().getOrDefault("name", super.getName());
Object name = getParams().get("name");
if (name!=null) {
return name.toString();
}
return super.getName();
}
}

17
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java
View File

@ -17,6 +17,9 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.List;
import java.util.Map;
/**
* A StmtsDef contains a list of rawStmts, as well as all of the optional
* block parameters that can be assigned to {@link BlockParams}, which includes
@ -24,4 +27,18 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml;
*/
public class RawStmtsBlock extends StatementsOwner {
public RawStmtsBlock() {}
public void setFieldsByReflection(Map<String, Object> propsmap) {
// Object stmtValues = propsmap.remove("statements");
// if (stmtValues!=null) {
// if (stmtValues instanceof List) {
// setByObject(stmtValues);
// } else {
// throw new RuntimeException("Invalid type for statements property: " + stmtValues.getClass().getCanonicalName());
// }
// }
super.setFieldsByReflection(propsmap);
}
}

37
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java
View File

@ -19,6 +19,8 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* A statements doc can have both a list of statement blocks and/or a
@ -35,7 +37,39 @@ public class RawStmtsDoc extends StatementsOwner {
private List<RawStmtsBlock> blocks = new ArrayList<>();
// no-args ctor is required
public RawStmtsDoc() {}
public RawStmtsDoc() {
}
public void setFieldsByReflection(Map<String, Object> properties) {
Object blocksObjects = properties.remove("blocks");
if (blocksObjects instanceof List) {
List<Object> blockList = ((List<Object>) blocksObjects);
for (Object blockData : blockList) {
if (blockData instanceof Map) {
Map<String,Object> blockDataMap = (Map<String,Object>)blockData;
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawStmtsBlock);
} else {
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
}
}
}
Object scenariosData = properties.remove("scenarios");
if (scenariosData!=null) {
if (scenariosData instanceof Map) {
RawScenarios rawScenarios = new RawScenarios();
Map<String,Object> scenariosObjMap = (Map<String,Object>)scenariosData;
rawScenarios.setPropertiesByReflection(scenariosObjMap);
} else {
throw new RuntimeException("Invalid type for scenarios data: " + scenariosData.getClass().getCanonicalName());
}
}
super.setFieldsByReflection(properties);
}
/**
* Return the list of statement blocks in this RawStmtsDoc.
@ -69,4 +103,5 @@ public class RawStmtsDoc extends StatementsOwner {
public void setScenarios(RawScenarios scenarios) {
this.scenarios = scenarios;
}
}

1
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java
View File

@ -89,6 +89,7 @@ public class RawYamlStatementLoader {
try {
Iterable<Object> objects = yaml.loadAll(data);
List<RawStmtsDoc> stmtListList = new ArrayList<>();
for (Object object : objects) {
RawStmtsDoc tgsd = (RawStmtsDoc) object;

17
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java
View File

@ -34,6 +34,21 @@ public class StatementsOwner extends BlockParams {
this.rawStmtDefs = rawStmtDefs;
}
public void setFieldsByReflection(Map<String, Object> propsmap) {
Object statementsObject = propsmap.remove("statements");
if (statementsObject!=null) {
setByObject(statementsObject);
}
// if (statementsObject!=null) {
// if (statementsObject instanceof List) {
// setByObject(statementsObject);
// } else {
// throw new RuntimeException("Invalid type for statements property: " + statementsObject.getClass().getCanonicalName());
// }
// }
super.setFieldsByReflection(propsmap);
}
@SuppressWarnings("unchecked")
public void setByObject(Object object) {
if (object instanceof List) {
@ -61,7 +76,7 @@ public class StatementsOwner extends BlockParams {
valueMap.put("name", entries.getKey());
itemizedMaps.add(valueMap);
} else if (value instanceof String) {
Map<String,Object> stmtDetails = new HashMap<String,Object>() {{
Map<String,Object> stmtDetails = new HashMap<>() {{
put("name", entries.getKey());
put("stmt", entries.getValue());
}};

15
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/Tags.java
View File

@ -25,7 +25,7 @@ import java.util.Map;
public class Tags implements Tagged {
private Map<String, String> tags = new LinkedHashMap<>();
private final Map<String, String> tags = new LinkedHashMap<>();
@Override
public Map<String, String> getTags() {
@ -37,4 +37,17 @@ public class Tags implements Tagged {
this.tags.putAll(tags);
}
@SuppressWarnings("unchecked")
public void setFieldsByReflection(Map<String, Object> propsmap) {
Object tagsValues = propsmap.remove("tags");
if (tagsValues != null) {
if (tagsValues instanceof Map) {
Map<Object, Object> tagsMap = (Map<Object, Object>) tagsValues;
tagsMap.forEach((ko, vo) -> tags.put(ko.toString(), vo.toString()));
} else {
throw new RuntimeException("Invalid type for tags property: " + tags.getClass().getCanonicalName());
}
}
}
}

87
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/stmtloader/GenericStmtLoader.java Normal file
View File

@ -0,0 +1,87 @@
package io.nosqlbench.engine.api.activityconfig.stmtloader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.DiscreteProbabilityBuffer;
import org.slf4j.Logger;
import org.yaml.snakeyaml.Yaml;
import java.util.*;
import java.util.function.Function;
import java.util.stream.StreamSupport;
public class GenericStmtLoader {
List<Function<String, String>> stringTransformers = new ArrayList<>();
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
public void addTransformers(Function<String, String>... newTransformers) {
Collections.addAll(this.transformers, newTransformers);
}
public RawStmtsDocList load(
Logger logger,
String path,
String... searchPaths) {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension("yaml").first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
} catch (Exception e) {
throw new RuntimeException("error while reading file " + path, e);
}
try {
if (logger != null) logger.debug("Applying string transformer to yaml data:" + data);
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
} catch (Exception e) {
RuntimeException t = new ActivityInitializationError("Error applying string applyTransforms to input", e);
if (logger != null) logger.error(t.getMessage(), t);
throw t;
}
return parseYaml(logger, data);
}
private RawStmtsDocList parseYaml(Logger logger, String data) {
Yaml yaml = new Yaml();
Iterable<Object> objects = yaml.loadAll(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
RawStmtsDoc doc = new RawStmtsDoc();
doc.setFieldsByReflection((Map<String, Object>) object);
newDocList.add(doc);
} else {
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
}
}
RawStmtsDocList rawStmtsDocList = new RawStmtsDocList(newDocList);
return rawStmtsDocList;
}
protected String applyTransforms(Logger logger, String data) {
for (Function<String, String> xform : stringTransformers) {
try {
if (logger != null) logger.debug("Applying string transformer to yaml data:" + xform);
data = xform.apply(data);
} catch (Exception e) {
RuntimeException t = new ActivityInitializationError("Error applying string applyTransforms to input", e);
if (logger != null) logger.error(t.getMessage(), t);
throw t;
}
}
return data;
}
}

54
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDef.java
View File

@ -22,7 +22,10 @@ import io.nosqlbench.engine.api.activityconfig.ParsedStmt;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.util.Tagged;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
public class StmtDef implements Tagged {
@ -43,15 +46,58 @@ public class StmtDef implements Tagged {
}
public Map<String,String> getBindings() {
return new MultiMapLookup(rawStmtDef.getBindings(), block.getBindings());
return new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings());
}
public Map<String, String> getParams() {
return new MultiMapLookup(rawStmtDef.getParams(), block.getParams());
public Map<String, Object> getParams() {
return new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
}
public <T> Map<String,T> getParamsAsValueType(Class<? extends T> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
Map<String,T> map = new LinkedHashMap<>();
lookup.forEach((k,v)->map.put(k,type.cast(v)));
return map;
}
@SuppressWarnings("unchecked")
public <V> V getParamOrDefault(String name, V defaultValue) {
Objects.requireNonNull(defaultValue);
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
if (!lookup.containsKey(name)) {
return defaultValue;
}
Object value = lookup.get(name);
return (V) defaultValue.getClass().cast(value);
}
public <V> V getParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
Object object = lookup.get(name);
V value = type.cast(object);
return value;
}
@SuppressWarnings("unchecked")
public <V> Optional<V> getOptionalParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
if (lookup.containsKey(name)) {
Object object = lookup.get(name);
if (object==null) {
return Optional.empty();
}
return Optional.of((V) type.cast(object));
}
return Optional.empty();
}
public Optional<String> getOptionalParam(String name) {
return getOptionalParam(name,String.class);
}
public Map<String,String> getTags() {
return new MultiMapLookup(rawStmtDef.getTags(), block.getTags());
return new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags());
}
@Override

44
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java
View File

@ -21,11 +21,9 @@ import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.util.Tagged;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
public class StmtsBlock implements Tagged, Iterable<StmtDef> {
@ -43,13 +41,13 @@ public class StmtsBlock implements Tagged, Iterable<StmtDef> {
}
public List<StmtDef> getStmts() {
List<StmtDef> rawStmtDefs = new ArrayList<>();
List<RawStmtDef> statements = rawStmtsBlock.getRawStmtDefs();
for (int i = 0; i < statements.size(); i++) {
rawStmtDefs.add(
new StmtDef(this,statements.get(i))
new StmtDef(this, statements.get(i))
);
}
return rawStmtDefs;
@ -69,18 +67,44 @@ public class StmtsBlock implements Tagged, Iterable<StmtDef> {
}
public Map<String, String> getTags() {
return new MultiMapLookup(rawStmtsBlock.getTags(), rawStmtsDoc.getTags());
return new MultiMapLookup<>(rawStmtsBlock.getTags(), rawStmtsDoc.getTags());
}
public Map<String, String> getParams() {
return new MultiMapLookup(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
public Map<String, Object> getParams() {
return new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
}
public Map<String, String> getParamsAsText() {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
LinkedHashMap<String, String> stringmap = new LinkedHashMap<>();
lookup.forEach((k, v) -> stringmap.put(k, v.toString()));
return stringmap;
}
@SuppressWarnings("unchecked")
public <V> V getParamOrDefault(String name, V defaultValue) {
Objects.requireNonNull(defaultValue);
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
if (!lookup.containsKey(name)) {
return defaultValue;
}
Object value = lookup.get(name);
return (V) defaultValue.getClass().cast(value);
}
public <V> V getParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
Object object = lookup.get(name);
V value = type.cast(object);
return value;
}
public Map<String, String> getBindings() {
return new MultiMapLookup(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings());
return new MultiMapLookup<>(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings());
}
@Override
@NotNull
public Iterator<StmtDef> iterator() {
return getStmts().iterator();
}

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java
View File

@ -68,7 +68,7 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
/**
* @return a usable map of parameters, including those inherited from the parent doc
*/
public Map<String, String> getParams() {
public Map<String, Object> getParams() {
return rawStmtsDoc.getParams();
}

55
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDefDefsTest.java
View File

@ -62,6 +62,39 @@ public class RawStmtDefDefsTest {
assertThat(block22.getTags().get("root1")).isEqualTo("value23");
}
@Test
public void testLayeringWithGenericLoader() {
StmtsDocList all = StatementsLoader.load(
StatementsLoader.Loader.generified,
logger,
"testdocs" + "/docs_blocks_stmts.yaml"
);
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
assertThat(doc1.getName()).isEqualTo("doc1");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsDoc doc2 = all.getStmtDocs().get(1);
assertThat(doc2.getBlocks()).hasSize(2);
StmtsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getBindings()).hasSize(2);
assertThat(block1.getName()).isEqualTo("doc1--block0");
assertThat(block1.getTags()).hasSize(1);
StmtsBlock block21 = doc2.getBlocks().get(0);
StmtsBlock block22 = doc2.getBlocks().get(1);
assertThat(block21.getName()).isEqualTo("doc2--block1");
assertThat(block21.getTags()).hasSize(3);
assertThat(block22.getName()).isEqualTo("doc2--block2");
assertThat(block22.getTags()).hasSize(2);
assertThat(block22.getTags().get("root1")).isEqualTo("value23");
}
@Test
public void testStatementRendering() {
StmtsDocList all = StatementsLoader.load(logger, "testdocs/docs_blocks_stmts.yaml");
@ -78,4 +111,26 @@ public class RawStmtDefDefsTest {
}
@Test
public void testStatementRenderingGenerified() {
StmtsDocList all = StatementsLoader.load(
StatementsLoader.Loader.generified,
logger,
"testdocs" +
"/docs_blocks_stmts.yaml"
);
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
StmtsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("doc1--block0");
List<StmtDef> assys = block1.getStmts();
assertThat(assys).hasSize(2);
StmtDef sdef1 = assys.get(0);
assertThat(sdef1.getName()).isEqualTo("doc1--block0--stmt1");
assertThat(assys.get(0).getStmt()).isEqualTo("s1");
}
}

2
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java
View File

@ -101,7 +101,7 @@ public class StmtsDocListTest {
StmtsBlock block0 = doc1.getBlocks().get(0);
Map<String, String> doc1block0tags = block0.getTags();
Map<String, String> doc1block0params = block0.getParams();
Map<String, String> doc1block0params = block0.getParamsAsText();
Map<String, String> doc1block0bindings = block0.getBindings();
assertThat(doc1block0tags).hasSize(3);