From 46b2447d818d2307650b797375829c5406c2078d Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 11 Jun 2020 12:21:44 -0500 Subject: [PATCH] #158 Simplify YamlStatement Loader --- .../activitytype/cql/core/CqlActivity.java | 55 +++-- .../activitytype/cqld4/core/CqlActivity.java | 213 +++++++++--------- .../cql/statements/CQLOptionsTest.java | 78 ------- .../cqlverify/CqlVerifyActivity.java | 4 +- .../activitytype/http/HttpActivity.java | 2 +- .../driver/mongodb/MongoActivity.java | 3 +- .../driver/mongodb/ReadyMongoStatement.java | 5 +- .../activitytype/stdout/StdoutActivity.java | 2 +- .../driver/webdriver/CommandTemplate.java | 2 +- .../driver/webdriver/WebDriverActivity.java | 2 +- .../api/activityconfig/MultiMapLookup.java | 32 +-- .../engine/api/activityconfig/ParsedStmt.java | 2 +- .../api/activityconfig/StatementsLoader.java | 47 +++- .../activityconfig/rawyaml/BlockParams.java | 59 ++++- .../activityconfig/rawyaml/RawScenarios.java | 6 +- .../activityconfig/rawyaml/RawStmtDef.java | 9 +- .../activityconfig/rawyaml/RawStmtsBlock.java | 17 ++ .../activityconfig/rawyaml/RawStmtsDoc.java | 37 ++- .../rawyaml/RawYamlStatementLoader.java | 1 + .../rawyaml/StatementsOwner.java | 17 +- .../api/activityconfig/rawyaml/Tags.java | 15 +- .../stmtloader/GenericStmtLoader.java | 87 +++++++ .../api/activityconfig/yaml/StmtDef.java | 54 ++++- .../api/activityconfig/yaml/StmtsBlock.java | 44 +++- .../api/activityconfig/yaml/StmtsDoc.java | 2 +- .../rawyaml/RawStmtDefDefsTest.java | 55 +++++ .../activityconfig/yaml/StmtsDocListTest.java | 2 +- 27 files changed, 583 insertions(+), 269 deletions(-) delete mode 100644 driver-cqld4/src/test/java/com/datastax/ebdrivers/cql/statements/CQLOptionsTest.java create mode 100644 engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/stmtloader/GenericStmtLoader.java diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java index 5b6e92406..9e61bb4cf 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java @@ -137,7 +137,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef private void initSequencer() { Session session = getSession(); - Map fconfig = Map.of("session",session); + Map fconfig = Map.of("session", session); SequencerType sequencerType = SequencerType.valueOf( getParams().getOptionalString("seq").orElse("bucket") @@ -160,33 +160,28 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef for (StmtDef stmtDef : stmts) { ParsedStmt parsed = stmtDef.getParsed().orError(); - boolean prepared = Boolean.valueOf(stmtDef.getParams().getOrDefault("prepared", "true")); - boolean parametrized = Boolean.valueOf(stmtDef.getParams().getOrDefault("parametrized", "false")); - long ratio = Long.valueOf(stmtDef.getParams().getOrDefault("ratio", "1")); + boolean prepared = stmtDef.getParamOrDefault("prepared", true); + boolean parametrized = stmtDef.getParamOrDefault("parametrized", false); + long ratio = stmtDef.getParamOrDefault("ratio", 1); - Optional cl = Optional.ofNullable( - stmtDef.getParams().getOrDefault("cl", null)).map(ConsistencyLevel::valueOf); - - Optional serial_cl = Optional.ofNullable( - stmtDef.getParams().getOrDefault("serial_cl", null)).map(ConsistencyLevel::valueOf); - - Optional idempotent = Optional.ofNullable(stmtDef.getParams().getOrDefault("idempotent", null)) - .map(Boolean::valueOf); + Optional cl = stmtDef.getOptionalParam("cl", String.class).map(ConsistencyLevel::valueOf); + Optional serial_cl = stmtDef.getOptionalParam("serial_cl").map(ConsistencyLevel::valueOf); + Optional idempotent = stmtDef.getOptionalParam("idempotent", boolean.class); StringBuilder psummary = new StringBuilder(); - boolean instrument = Optional.ofNullable(stmtDef.getParams() - .get("instrument")).map(Boolean::valueOf) + boolean instrument = stmtDef.getOptionalParam("instrument", boolean.class) .orElse(getParams().getOptionalBoolean("instrument").orElse(false)); - String logresultcsv = stmtDef.getParams().getOrDefault("logresultcsv",""); + String logresultcsv = stmtDef.getParamOrDefault("logresultcsv", ""); + String logresultcsv_act = getParams().getOptionalString("logresultcsv").orElse(""); if (!logresultcsv_act.isEmpty() && !logresultcsv_act.toLowerCase().equals("true")) { throw new RuntimeException("At the activity level, only logresultcsv=true is allowed, no other values."); } logresultcsv = !logresultcsv.isEmpty() ? logresultcsv : logresultcsv_act; - logresultcsv = !logresultcsv.toLowerCase().equals("true") ? logresultcsv : stmtDef.getName()+"--results.csv"; + logresultcsv = !logresultcsv.toLowerCase().equals("true") ? logresultcsv : stmtDef.getName() + "--results.csv"; logger.debug("readying statement[" + (prepared ? "" : "un") + "prepared]:" + parsed.getStmt()); @@ -207,11 +202,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef psummary.append(" idempotent=").append(idempotent); prepare.setIdempotent(i); }); - CqlBinderTypes binderType = CqlBinderTypes.valueOf(stmtDef.getParams() - .getOrDefault("binder", CqlBinderTypes.DEFAULT.toString())); + + CqlBinderTypes binderType = stmtDef.getOptionalParam("binder") + .map(CqlBinderTypes::valueOf) + .orElse(CqlBinderTypes.DEFAULT); template = new ReadyCQLStatementTemplate(fconfig, binderType, getSession(), prepare, ratio, - parsed.getName()); + parsed.getName()); } else { SimpleStatement simpleStatement = new SimpleStatement(stmtForDriver); cl.ifPresent((conlvl) -> { @@ -227,18 +224,18 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef simpleStatement.setIdempotent(i); }); template = new ReadyCQLStatementTemplate(fconfig, getSession(), simpleStatement, ratio, - parsed.getName(), parametrized); + parsed.getName(), parametrized); } - Optional.ofNullable(stmtDef.getParams().getOrDefault("save", null)) - .map(s -> s.split("[,; ]")) + stmtDef.getOptionalParam("save") + .map(s -> s.split("[,: ]")) .map(Save::new) .ifPresent(save_op -> { psummary.append(" save=>").append(save_op.toString()); template.addRowCycleOperators(save_op); }); - Optional.ofNullable(stmtDef.getParams().getOrDefault("rsoperators", null)) + stmtDef.getOptionalParam("rsoperators") .map(s -> s.split(",")) .stream().flatMap(Arrays::stream) .map(ResultSetCycleOperators::newOperator) @@ -247,7 +244,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef template.addResultSetOperators(rso); }); - Optional.ofNullable(stmtDef.getParams().getOrDefault("rowoperators", null)) + stmtDef.getOptionalParam("rowoperators") .map(s -> s.split(",")) .stream().flatMap(Arrays::stream) .map(RowCycleOperators::newOperator) @@ -264,7 +261,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef if (!logresultcsv.isEmpty()) { logger.info("Adding per-statement result CSV logging to statement '" + parsed.getName() + "'"); - template.logResultCsv(this,logresultcsv); + template.logResultCsv(this, logresultcsv); psummary.append(" logresultcsv=>").append(logresultcsv); } @@ -357,7 +354,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef rawblock.setTags(rawdef.getTags()); // params - Map params = new HashMap<>(rawdef.getParams()); + Map params = new HashMap<>(rawdef.getParams()); if (rawstmt.getConsistencyLevel() != null && !rawstmt.getConsistencyLevel().isEmpty()) params.put("cl", rawstmt.getConsistencyLevel()); if (!rawstmt.isPrepared()) params.put("prepared", "false"); @@ -516,7 +513,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef if (keyval.length == 1) { String verb = keyval[0]; ErrorResponse errorResponse = getErrorResponseOrBasicError(verb); - newerrorHandler.setDefaultHandler( + newerrorHandler.setDefaultHandler( new NBCycleErrorHandler( errorResponse, exceptionCountMetrics, @@ -558,8 +555,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef private ErrorResponse getErrorResponseOrBasicError(String verb) { try { return ErrorResponse.valueOf(verb); - }catch(IllegalArgumentException e){ - throw new BasicError("Invalid parameter for errors: '"+ verb + "' should be one of: " + StringUtils.join(ErrorResponse.values(), ", ")); + } catch (IllegalArgumentException e) { + throw new BasicError("Invalid parameter for errors: '" + verb + "' should be one of: " + StringUtils.join(ErrorResponse.values(), ", ")); } } diff --git a/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/core/CqlActivity.java b/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/core/CqlActivity.java index 859f20048..b094233d9 100644 --- a/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/core/CqlActivity.java +++ b/driver-cqld4/src/main/java/io/nosqlbench/activitytype/cqld4/core/CqlActivity.java @@ -143,7 +143,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef Map fconfig = Map.of("session", session); SequencerType sequencerType = SequencerType.valueOf( - getParams().getOptionalString("seq").orElse("bucket") + getParams().getOptionalString("seq").orElse("bucket") ); SequencePlanner planner = new SequencePlanner<>(sequencerType); @@ -163,17 +163,19 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef for (StmtDef stmtDef : stmts) { ParsedStmt parsed = stmtDef.getParsed().orError(); - boolean prepared = Boolean.valueOf(stmtDef.getParams().getOrDefault("prepared", "true")); - boolean parametrized = Boolean.valueOf(stmtDef.getParams().getOrDefault("parametrized", "false")); - long ratio = Long.valueOf(stmtDef.getParams().getOrDefault("ratio", "1")); + + boolean prepared = stmtDef.getParamOrDefault("prepared", true); + boolean parametrized = stmtDef.getParamOrDefault("parametrized", false); + long ratio = stmtDef.getParamOrDefault("ratio", 1); StringBuilder psummary = new StringBuilder(); - boolean instrument = Optional.ofNullable(stmtDef.getParams() - .get("instrument")).map(Boolean::valueOf) - .orElse(getParams().getOptionalBoolean("instrument").orElse(false)); + boolean instrument = stmtDef.getOptionalParam("instrument") + .or(() -> getParams().getOptionalString("instrument")) + .map(Boolean::parseBoolean) + .orElse(false); - String logresultcsv = stmtDef.getParams().getOrDefault("logresultcsv", ""); + String logresultcsv = stmtDef.getParamOrDefault("logresultcsv",""); String logresultcsv_act = getParams().getOptionalString("logresultcsv").orElse(""); if (!logresultcsv_act.isEmpty() && !logresultcsv_act.toLowerCase().equals("true")) { @@ -190,77 +192,79 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef SimpleStatementBuilder stmtBuilder = SimpleStatement.builder(stmtForDriver); psummary.append(" statement=>").append(stmtForDriver); - Optional.ofNullable(stmtDef.getParams().getOrDefault("cl", null)) - .map(DefaultConsistencyLevel::valueOf) - .map(conlvl -> { - psummary.append(" consistency_level=>").append(conlvl); - return conlvl; - }) - .ifPresent(stmtBuilder::setConsistencyLevel); - Optional.ofNullable(stmtDef.getParams().getOrDefault("serial_cl", null)) - .map(DefaultConsistencyLevel::valueOf) - .map(sconlvel -> { - psummary.append(" serial_consistency_level=>").append(sconlvel); - return sconlvel; - }) - .ifPresent(stmtBuilder::setSerialConsistencyLevel); + stmtDef.getOptionalParam("cl") + .map(DefaultConsistencyLevel::valueOf) + .map(conlvl -> { + psummary.append(" consistency_level=>").append(conlvl); + return conlvl; + }) + .ifPresent(stmtBuilder::setConsistencyLevel); - Optional.ofNullable(stmtDef.getParams().getOrDefault("idempotent", null)) - .map(Boolean::valueOf) - .map(idempotent -> { - psummary.append(" idempotent=").append(idempotent); - return idempotent; - }) - .ifPresent(stmtBuilder::setIdempotence); + stmtDef.getOptionalParam("serial_cl") + .map(DefaultConsistencyLevel::valueOf) + .map(sconlvel -> { + psummary.append(" serial_consistency_level=>").append(sconlvel); + return sconlvel; + }) + .ifPresent(stmtBuilder::setSerialConsistencyLevel); + + stmtDef.getOptionalParam("idempotent") + .map(Boolean::valueOf) + .map(idempotent -> { + psummary.append(" idempotent=").append(idempotent); + return idempotent; + }) + .ifPresent(stmtBuilder::setIdempotence); if (prepared) { PreparedStatement preparedStatement = getSession().prepare(stmtBuilder.build()); - CqlBinderTypes binderType = CqlBinderTypes.valueOf(stmtDef.getParams() - .getOrDefault("binder", CqlBinderTypes.DEFAULT.toString())); + CqlBinderTypes binderType = stmtDef.getOptionalParam("binder") + .map(CqlBinderTypes::valueOf) + .orElse(CqlBinderTypes.DEFAULT); template = new ReadyCQLStatementTemplate( - fconfig, - binderType, - getSession(), - preparedStatement, - ratio, - parsed.getName() + fconfig, + binderType, + getSession(), + preparedStatement, + ratio, + parsed.getName() ); } else { SimpleStatement simpleStatement = SimpleStatement.newInstance(stmtForDriver); template = new ReadyCQLStatementTemplate(fconfig, getSession(), simpleStatement, ratio, - parsed.getName(), parametrized); + parsed.getName(), parametrized); } - Optional.ofNullable(stmtDef.getParams().getOrDefault("save", null)) - .map(s -> s.split("[,; ]")) - .map(Save::new) - .ifPresent(save_op -> { - psummary.append(" save=>").append(save_op.toString()); - template.addRowCycleOperators(save_op); - }); + stmtDef.getOptionalParam("save") + .map(s -> s.split("[,; ]")) + .map(Save::new) + .ifPresent(save_op -> { + psummary.append(" save=>").append(save_op.toString()); + template.addRowCycleOperators(save_op); + }); - Optional.ofNullable(stmtDef.getParams().getOrDefault("rsoperators", null)) - .map(s -> s.split(",")) - .stream().flatMap(Arrays::stream) - .map(ResultSetCycleOperators::newOperator) - .forEach(rso -> { - psummary.append(" rsop=>").append(rso.toString()); - template.addResultSetOperators(rso); - }); + stmtDef.getOptionalParam("rsoperators") + .map(s -> s.split(",")) + .stream().flatMap(Arrays::stream) + .map(ResultSetCycleOperators::newOperator) + .forEach(rso -> { + psummary.append(" rsop=>").append(rso.toString()); + template.addResultSetOperators(rso); + }); - Optional.ofNullable(stmtDef.getParams().getOrDefault("rowoperators", null)) - .map(s -> s.split(",")) - .stream().flatMap(Arrays::stream) - .map(RowCycleOperators::newOperator) - .forEach(ro -> { - psummary.append(" rowop=>").append(ro.toString()); - template.addRowCycleOperators(ro); - }); + stmtDef.getOptionalParam("rowoperators") + .map(s -> s.split(",")) + .stream().flatMap(Arrays::stream) + .map(RowCycleOperators::newOperator) + .forEach(ro -> { + psummary.append(" rowop=>").append(ro.toString()); + template.addRowCycleOperators(ro); + }); if (instrument) { logger.info("Adding per-statement success and error and resultset-size timers to statement '" + parsed.getName() + "'"); @@ -305,9 +309,9 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef case "1": doclist = getVersion1StmtsDoc(interp, yaml_loc); logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + " with compatibility mode. " + - "This will be deprecated in a future release."); + "This will be deprecated in a future release."); logger.warn("DEPRECATED-FORMAT: Please refer to " + - "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); + "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); break; case "2": doclist = StatementsLoader.load(logger, yaml_loc, interp, "activities"); @@ -315,22 +319,22 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef case "unset": try { logger.debug("You can suffix your yaml filename or url with the " + - "format version, such as :1 or :2. Assuming version 2."); + "format version, such as :1 or :2. Assuming version 2."); doclist = StatementsLoader.load(null, yaml_loc, interp, "activities"); } catch (Exception ignored) { try { doclist = getVersion1StmtsDoc(interp, yaml_loc); logger.warn("DEPRECATED-FORMAT: Loaded yaml " + yaml_loc + - " with compatibility mode. This will be deprecated in a future release."); + " with compatibility mode. This will be deprecated in a future release."); logger.warn("DEPRECATED-FORMAT: Please refer to " + - "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); + "http://docs.engineblock.io/user-guide/standard_yaml/ for more details."); } catch (Exception compatError) { logger.warn("Tried to load yaml in compatibility mode, " + - "since it failed to load with the standard format, " + - "but found an error:" + compatError); + "since it failed to load with the standard format, " + + "but found an error:" + compatError); logger.warn("The following detailed errors are provided only " + - "for the standard format. To force loading version 1 with detailed logging, add" + - " a version qualifier to your yaml filename or url like ':1'"); + "for the standard format. To force loading version 1 with detailed logging, add" + + " a version qualifier to your yaml filename or url like ':1'"); // retrigger the error again, this time with logging enabled. doclist = StatementsLoader.load(logger, yaml_loc, interp, "activities"); } @@ -338,7 +342,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef break; default: throw new RuntimeException("Unrecognized yaml format version, expected :1 or :2 " + - "at end of yaml file, but got " + yamlVersion + " instead."); + "at end of yaml file, but got " + yamlVersion + " instead."); } return doclist; @@ -363,12 +367,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef rawblock.setTags(rawdef.getTags()); // params - Map params = new HashMap<>(rawdef.getParams()); + Map params = new HashMap<>(rawdef.getParams()); if (rawstmt.getConsistencyLevel() != null && !rawstmt.getConsistencyLevel().isEmpty()) params.put("cl", rawstmt.getConsistencyLevel()); if (!rawstmt.isPrepared()) params.put("prepared", "false"); if (rawstmt.getRatio() != 1L) params.put("ratio", String.valueOf(rawstmt.getRatio())); + rawblock.setParams(params); @@ -401,10 +406,10 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef @Override public String toString() { return "CQLActivity {" + - "activityDef=" + activityDef + - ", session=" + session + - ", opSequence=" + this.opsequence + - '}'; + "activityDef=" + activityDef + + ", session=" + session + + ", opSequence=" + this.opsequence + + '}'; } @Override @@ -420,7 +425,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef if (fetchSizeOption.isPresent()) { int fetchSize = fetchSizeOption.flatMap(Unit::bytesFor).map(Double::intValue).orElseThrow(() -> new RuntimeException( - "Unable to parse fetch size from " + fetchSizeOption.get() + "Unable to parse fetch size from " + fetchSizeOption.get() )); if (fetchSize > 10000000 && fetchSize < 1000000000) { logger.warn("Setting the fetchsize to " + fetchSize + " is unlikely to give good performance."); @@ -429,7 +434,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef } logger.trace("setting fetchSize to " + fetchSize); - CQLSessionCache.get().getSession(activityDef).set(DefaultDriverOption.REQUEST_PAGE_SIZE,fetchSize); + CQLSessionCache.get().getSession(activityDef).set(DefaultDriverOption.REQUEST_PAGE_SIZE, fetchSize); } this.retryDelay = params.getOptionalLong("retrydelay").orElse(0L); @@ -440,8 +445,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef this.maxpages = params.getOptionalInteger("maxpages").orElse(1); this.statementFilter = params.getOptionalString("tokens") - .map(s -> new TokenRangeStmtFilter(getSession(), s)) - .orElse(null); + .map(s -> new TokenRangeStmtFilter(getSession(), s)) + .orElse(null); if (statementFilter != null) { logger.info("filtering statements" + statementFilter); @@ -450,13 +455,13 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef errorHandler = configureErrorHandler(); params.getOptionalString("trace") - .map(SimpleConfig::new) - .map(TraceLogger::new) - .ifPresent( - tl -> { - addResultSetCycleOperator(tl); - addStatementModifier(tl); - }); + .map(SimpleConfig::new) + .map(TraceLogger::new) + .ifPresent( + tl -> { + addResultSetCycleOperator(tl); + addStatementModifier(tl); + }); this.maxTotalOpsInFlight = params.getOptionalLong("async").orElse(1L); @@ -514,8 +519,8 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef HashedCQLErrorHandler newerrorHandler = new HashedCQLErrorHandler(exceptionCountMetrics); String errors = activityDef.getParams() - .getOptionalString("errors") - .orElse("stop,retryable->retry,unverified->stop"); + .getOptionalString("errors") + .orElse("stop,retryable->retry,unverified->stop"); String[] handlerSpecs = errors.split(","); @@ -524,31 +529,31 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef if (keyval.length == 1) { String verb = keyval[0]; newerrorHandler.setDefaultHandler( - new NBCycleErrorHandler( - ErrorResponse.valueOf(verb), - exceptionCountMetrics, - exceptionHistoMetrics, - getParams().getOptionalLong("async").isEmpty() - ) + new NBCycleErrorHandler( + ErrorResponse.valueOf(verb), + exceptionCountMetrics, + exceptionHistoMetrics, + getParams().getOptionalLong("async").isEmpty() + ) ); } else { String pattern = keyval[0]; String verb = keyval[1]; if (newerrorHandler.getGroupNames().contains(pattern)) { NBCycleErrorHandler handler = - new NBCycleErrorHandler( - ErrorResponse.valueOf(verb), - exceptionCountMetrics, - exceptionHistoMetrics, - getParams().getOptionalLong("async").isEmpty() - ); + new NBCycleErrorHandler( + ErrorResponse.valueOf(verb), + exceptionCountMetrics, + exceptionHistoMetrics, + getParams().getOptionalLong("async").isEmpty() + ); logger.info("Handling error group '" + pattern + "' with handler:" + handler); newerrorHandler.setHandlerForGroup(pattern, handler); } else { NBCycleErrorHandler handler = new NBCycleErrorHandler( - ErrorResponse.valueOf(keyval[1]), - exceptionCountMetrics, - exceptionHistoMetrics, + ErrorResponse.valueOf(keyval[1]), + exceptionCountMetrics, + exceptionHistoMetrics, getParams().getOptionalLong("async").isEmpty() ); logger.info("Handling error pattern '" + pattern + "' with handler:" + handler); diff --git a/driver-cqld4/src/test/java/com/datastax/ebdrivers/cql/statements/CQLOptionsTest.java b/driver-cqld4/src/test/java/com/datastax/ebdrivers/cql/statements/CQLOptionsTest.java deleted file mode 100644 index b1d8f5433..000000000 --- a/driver-cqld4/src/test/java/com/datastax/ebdrivers/cql/statements/CQLOptionsTest.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.datastax.ebdrivers.cql.statements; - -import com.datastax.driver.core.HostDistance; -import com.datastax.driver.core.PoolingOptions; -import com.datastax.driver.core.SocketOptions; -import com.datastax.driver.core.policies.LoadBalancingPolicy; -import com.datastax.driver.core.policies.ReconnectionPolicy; -import com.datastax.driver.core.policies.SpeculativeExecutionPolicy; -import io.nosqlbench.activitytype.cqld4.core.CQLOptions; -import org.junit.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class CQLOptionsTest { - - @Test - public void testSpeculative() { - SpeculativeExecutionPolicy p1 = CQLOptions.speculativeFor("p99:5"); - assertThat(p1).isNotNull(); - SpeculativeExecutionPolicy p2 = CQLOptions.speculativeFor("p99:5:5000ms"); - assertThat(p2).isNotNull(); - } - - @Test - public void testConstant() { - SpeculativeExecutionPolicy p1 = CQLOptions.speculativeFor("5000ms:5"); - assertThat(p1).isNotNull(); - } - - @Test - public void testWhitelist() { - LoadBalancingPolicy lbp = CQLOptions.whitelistFor("127.0.0.1,127.0.0.2:123", null); - assertThat(lbp).isNotNull(); - } - - @Test - public void testReconnectPolicyPatterns() { - ReconnectionPolicy rp = CQLOptions.reconnectPolicyFor("exponential(123,321)"); - rp = CQLOptions.reconnectPolicyFor("constant(123)"); - - } - - @Test - public void testSocketOptionPatterns() { - SocketOptions so = CQLOptions.socketOptionsFor("read_timeout_ms=23423,connect_timeout_ms=2344;keep_alive:true,reuse_address:true;so_linger:323;tcp_no_delay=true;receive_buffer_size:100,send_buffer_size=1000"); - assertThat(so.getConnectTimeoutMillis()).isEqualTo(2344); - assertThat(so.getKeepAlive()).isEqualTo(true); - assertThat(so.getReadTimeoutMillis()).isEqualTo(23423); - assertThat(so.getReceiveBufferSize()).isEqualTo(100); - assertThat(so.getReuseAddress()).isEqualTo(true); - assertThat(so.getSendBufferSize()).isEqualTo(1000); - assertThat(so.getSoLinger()).isEqualTo(323); - assertThat(so.getTcpNoDelay()).isEqualTo(true); - - } - - @Test - public void testConnectionsPatterns() { - PoolingOptions po = CQLOptions.poolingOptionsFor("2345"); - assertThat(po.getCoreConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(2345); - assertThat(po.getMaxConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(Integer.MIN_VALUE); - assertThat(po.getMaxRequestsPerConnection(HostDistance.LOCAL)).isEqualTo(Integer.MIN_VALUE); - - PoolingOptions po2 = CQLOptions.poolingOptionsFor("1:2:3,4:5:6"); - assertThat(po2.getCoreConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(1); - assertThat(po2.getMaxConnectionsPerHost(HostDistance.LOCAL)).isEqualTo(2); - assertThat(po2.getMaxRequestsPerConnection(HostDistance.LOCAL)).isEqualTo(3); - assertThat(po2.getCoreConnectionsPerHost(HostDistance.REMOTE)).isEqualTo(4); - assertThat(po2.getMaxConnectionsPerHost(HostDistance.REMOTE)).isEqualTo(5); - assertThat(po2.getMaxRequestsPerConnection(HostDistance.REMOTE)).isEqualTo(6); - - PoolingOptions po3 = CQLOptions.poolingOptionsFor("1:2:3,4:5:6,heartbeat_interval_s:100,idle_timeout_s:123,pool_timeout_ms:234"); - assertThat(po3.getIdleTimeoutSeconds()).isEqualTo(123); - assertThat(po3.getPoolTimeoutMillis()).isEqualTo(234); - assertThat(po3.getHeartbeatIntervalSeconds()).isEqualTo(100); - - } -} diff --git a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java b/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java index 195dc4064..d20f479a9 100644 --- a/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java +++ b/driver-cqlverify/src/main/java/io/nosqlbench/activitytype/cqlverify/CqlVerifyActivity.java @@ -55,8 +55,8 @@ public class CqlVerifyActivity extends CqlActivity { Map bindings = stmts.get(0).getBindings(); if (stmts.get(0).getParams().containsKey("verify-fields")) { List fields = new ArrayList<>(); - String fieldsSpec = stmts.get(0).getParams().get("verify-fields"); - String[] vfields = fieldsSpec.split("\\s*,\\s*"); + String fieldSpec= stmts.get(0).getParamOrDefault("verify-fields",""); + String[] vfields = fieldSpec.split("\\s*,\\s*"); for (String vfield : vfields) { if (vfield.equals("*")) { bindings.forEach((k,v)->fields.add(k)); diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java index 92efdf41b..836d13afb 100644 --- a/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpActivity.java @@ -109,7 +109,7 @@ public class HttpActivity extends SimpleActivity implements Activity, ActivityDe StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt(), bt); StringBindings sb = sbt.resolve(); - sequencer.addOp(sb,Long.valueOf(stmt.getParams().getOrDefault("ratio","1"))); + sequencer.addOp(sb,stmt.getParamOrDefault("ratio",1)); } } else { logger.error("Unable to create an HTTP statement if no bindings or statements are defined."); diff --git a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java index ad553cf79..c2b5b3688 100644 --- a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java +++ b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java @@ -114,8 +114,7 @@ public class MongoActivity extends SimpleActivity implements ActivityDefObserver String statement = parsed.getPositionalStatement(Function.identity()); Objects.requireNonNull(statement); - sequencer.addOp(new ReadyMongoStatement(stmt), - Long.parseLong(stmt.getParams().getOrDefault("ratio","1"))); + sequencer.addOp(new ReadyMongoStatement(stmt), stmt.getParamOrDefault("ratio",1)); } return sequencer.resolve(); diff --git a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java index 9c974d51e..61cce3d16 100644 --- a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java +++ b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java @@ -20,8 +20,9 @@ public class ReadyMongoStatement { StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt(), paramBindings); this.bindings = template.resolve(); - this.readPreference = ReadPreference.valueOf(stmtDef.getParams() - .getOrDefault("readPreference","primary")); + this.readPreference = stmtDef.getOptionalParam("readPreference") + .map(ReadPreference::valueOf) + .orElse(ReadPreference.primary()); } public ReadPreference getReadPreference() { diff --git a/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java b/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java index 3c78d0a97..081285ed0 100644 --- a/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java +++ b/driver-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java @@ -167,7 +167,7 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt(), bt); StringBindings sb = sbt.resolve(); - sequencer.addOp(sb,Long.parseLong(stmt.getParams().getOrDefault("ratio","1"))); + sequencer.addOp(sb,stmt.getParamOrDefault("ratio",1)); } } else { logger.error("Unable to create a stdout statement if you have no active statements or bindings configured."); diff --git a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java index e29063623..3def7a47f 100644 --- a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java +++ b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/CommandTemplate.java @@ -30,7 +30,7 @@ public class CommandTemplate { this.name = stmt.getName(); String prefixed = "command=" + stmt.getStmt(); Map cmdMap = ParamsParser.parse(prefixed, canonicalize); - Map paramsMap = stmt.getParams(); + Map paramsMap = stmt.getParamsAsValueType(String.class); paramsMap.forEach((k,v) -> { if (cmdMap.containsKey(k)) { logger.warn("command property override: '" + k + "' superseded by param form with value '" + v + "'"); diff --git a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java index a77a6a116..e64c4036c 100644 --- a/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java +++ b/driver-web/src/main/java/io/nosqlbench/driver/webdriver/WebDriverActivity.java @@ -184,7 +184,7 @@ public class WebDriverActivity extends SimpleActivity { } for (StmtDef optemplate : stmts) { - long ratio = Long.parseLong(optemplate.getParams().getOrDefault("ratio", "1")); + long ratio = optemplate.getParamOrDefault("ratio", 1); CommandTemplate cmd = new CommandTemplate(optemplate, false); planner.addOp(cmd, ratio); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/MultiMapLookup.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/MultiMapLookup.java index 5ac379039..068f66c1f 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/MultiMapLookup.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/MultiMapLookup.java @@ -17,22 +17,24 @@ package io.nosqlbench.engine.api.activityconfig; +import org.jetbrains.annotations.NotNull; + import java.util.*; import java.util.stream.Collectors; -public class MultiMapLookup implements Map { +public class MultiMapLookup implements Map { - private final List> maps = new ArrayList<>(); + private final List> maps = new ArrayList<>(); public MultiMapLookup() { } - public MultiMapLookup(Map map1, Map map2) { + public MultiMapLookup(Map map1, Map map2) { add(map1); add(map2); } - public MultiMapLookup add(Map map) { + public MultiMapLookup add(Map map) { maps.add(map); return this; } @@ -59,26 +61,26 @@ public class MultiMapLookup implements Map { } @Override - public String get(Object key) { + public V get(Object key) { return maps.stream() .filter(m -> m.containsKey(String.valueOf(key))) .findFirst() - .map(m -> String.valueOf(m.get(key))) + .map(m -> m.get(key)) .orElse(null); } @Override - public String put(String key, String value) { + public V put(String key, V value) { throw immutable(); } @Override - public String remove(Object key) { + public V remove(Object key) { throw immutable(); } @Override - public void putAll(Map m) { + public void putAll(Map m) { throw immutable(); } @@ -88,6 +90,7 @@ public class MultiMapLookup implements Map { } @Override + @NotNull public Set keySet() { Set keys = new HashSet<>(); maps.stream().map(Map::keySet).flatMap(Set::stream) @@ -96,18 +99,19 @@ public class MultiMapLookup implements Map { } @Override - public Collection values() { + @NotNull + public Collection values() { return entrySet().stream() .map(Entry::getValue) .collect(Collectors.toList()); } @Override - public Set> entrySet() { - Map compositeMap = new HashMap<>(); + public Set> entrySet() { + Map compositeMap = new HashMap<>(); - for (Map map : maps) { - for (Entry entry : map.entrySet()) { + for (Map map : maps) { + for (Entry entry : map.entrySet()) { if (!compositeMap.containsKey(entry.getKey())) { compositeMap.put(entry.getKey(), entry.getValue()); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java index 0e2f5e73b..fed3e8a09 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmt.java @@ -153,7 +153,7 @@ public class ParsedStmt { /** * @return the params from the enclosed {@link StmtDef} */ - public Map getParams() { + public Map getParams() { return stmtDef.getParams(); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java index 2a8133368..5636ca32b 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java @@ -17,8 +17,10 @@ package io.nosqlbench.engine.api.activityconfig; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlStatementLoader; +import io.nosqlbench.engine.api.activityconfig.stmtloader.GenericStmtLoader; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.nb.api.content.Content; import io.nosqlbench.nb.api.content.NBIO; @@ -28,6 +30,11 @@ import java.util.function.Function; public class StatementsLoader { + public enum Loader { + original, + generified + } + public static StmtsDocList load(Logger logger, Content content) { RawYamlStatementLoader loader = new RawYamlStatementLoader(); RawStmtsDocList rawDocList = loader.loadString(logger, content.get()); @@ -42,18 +49,32 @@ public class StatementsLoader { // return layered; // } + public static StmtsDocList load(Loader loader, Logger logger, String path, String... searchPaths) { + RawStmtsDocList list= null; + + switch (loader) { + case generified: + GenericStmtLoader gloaderImpl = new GenericStmtLoader(); + list = gloaderImpl.load(logger,path,searchPaths); + case original: + RawYamlStatementLoader yloaderImpl = new RawYamlStatementLoader(); + list = yloaderImpl.load(logger, path, searchPaths); + } + return new StmtsDocList(list); + } + public static StmtsDocList load(Logger logger, String path, String... searchPaths) { Content content = NBIO.all() - .prefix(searchPaths) - .name(path) - .one(); + .prefix(searchPaths) + .name(path) + .one(); try { RawYamlStatementLoader loader = new RawYamlStatementLoader(); RawStmtsDocList rawDocList = loader.loadString(logger, content.get()); StmtsDocList layered = new StmtsDocList(rawDocList); return layered; } catch (Exception e) { - throw new RuntimeException("error while reading file " + path,e); + throw new RuntimeException("error while reading file " + path, e); } } @@ -64,4 +85,22 @@ public class StatementsLoader { return layered; } + public static StmtsDocList load(Loader loader, Logger logger, String path, + Function transformer, String... searchPaths) { + RawStmtsDocList list= null; + + switch (loader) { + case generified: + GenericStmtLoader gloaderImpl = new GenericStmtLoader(); + gloaderImpl.addTransformers(transformer); + list = gloaderImpl.load(logger,path,searchPaths); + case original: + RawYamlStatementLoader yloaderImpl = new RawYamlStatementLoader(); + yloaderImpl.addTransformer(transformer); + list = yloaderImpl.load(logger, path, searchPaths); + } + return new StmtsDocList(list); + } + + } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BlockParams.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BlockParams.java index f4720172c..7a7eaddb2 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BlockParams.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BlockParams.java @@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -25,10 +26,11 @@ public class BlockParams extends Tags { private String name = ""; private String desc = ""; - private Map bindings = new LinkedHashMap<>(); - private Map params = new LinkedHashMap<>(); + private final Map bindings = new LinkedHashMap<>(); + private final Map params = new LinkedHashMap<>(); - public BlockParams() {} + public BlockParams() { + } public String getDesc() { return desc; @@ -37,6 +39,7 @@ public class BlockParams extends Tags { public void setDesc(String desc) { this.desc = desc; } + public void setDescription(String desc) { this.desc = desc; } @@ -58,11 +61,17 @@ public class BlockParams extends Tags { this.bindings.putAll(bindings); } - public Map getParams() { - return params; + public Map getParams() { + return this.params; } - public void setParams(Map config) { +// public Map getParamsAsText() { +// Map paramsMap = new HashMap<>(); +// this.params.forEach((ko,vo) -> paramsMap.put(ko,vo.toString())); +// return paramsMap; +// } + + public void setParams(Map config) { this.params.clear(); this.params.putAll(config); } @@ -73,4 +82,40 @@ public class BlockParams extends Tags { setTags(other.getTags()); setParams(other.getParams()); } -} + + @SuppressWarnings("unchecked") + public void setFieldsByReflection(Map propsmap) { + + Object descriptionObj = propsmap.remove("description"); + if (descriptionObj!=null) { + setDescription(descriptionObj.toString()); + } + + Object nameObj = propsmap.remove("name"); + if (nameObj!=null) { + setName(nameObj.toString()); + } + + Object bindingsObject = propsmap.remove("bindings"); + if (bindingsObject!=null) { + if (bindingsObject instanceof Map) { + Map bindingsMap = (Map) bindingsObject; + bindingsMap.forEach((ko,vo) -> bindings.put(ko.toString(), vo.toString())); + } else { + throw new RuntimeException("Invalid type for bindings object: " + bindingsObject.getClass().getCanonicalName()); + } + } + + Object paramsObject = propsmap.remove("params"); + if (paramsObject!=null) { + if (paramsObject instanceof Map) { + Map paramsMap = (Map) paramsObject; + paramsMap.forEach((ko,vo) -> params.put(ko.toString(),vo)); + } else { + throw new RuntimeException("Invalid type for params object:" + paramsObject.getClass().getCanonicalName()); + } + } + + super.setFieldsByReflection(propsmap); + } +} \ No newline at end of file diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawScenarios.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawScenarios.java index 9e38ff0c0..1bd654530 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawScenarios.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawScenarios.java @@ -2,7 +2,7 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; import java.util.*; -public class RawScenarios extends LinkedHashMap> { +public class RawScenarios extends LinkedHashMap { public static String STEPNAME = "%03d"; @@ -10,6 +10,10 @@ public class RawScenarios extends LinkedHashMap> { return new LinkedList<>(this.keySet()); } + public void setPropertiesByReflection(Map scenarioInfo) { + scenarioInfo.forEach(this::put); + } + public Map getNamedScenario(String scenarioName) { Object v = this.get(scenarioName); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java index 2dedb5ea6..2cfa5d1c1 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java @@ -33,6 +33,7 @@ public class RawStmtDef extends BlockParams { this.statement = statement; } + @SuppressWarnings("unchecked") public RawStmtDef(String defaultName, Map map) { @@ -44,7 +45,7 @@ public class RawStmtDef extends BlockParams { Optional.ofNullable((Map) map.remove("tags")).ifPresent(this::setTags); Optional.ofNullable((Map) map.remove("bindings")).ifPresent(this::setBindings); - Optional.ofNullable((Map) map.remove("params")).ifPresent(this::setParams); + Optional.ofNullable((Map) map.remove("params")).ifPresent(this::setParams); // Depends on order stability, relying on LinkedHashMap -- Needs stability unit tests @@ -85,6 +86,10 @@ public class RawStmtDef extends BlockParams { } public String getName() { - return getParams().getOrDefault("name", super.getName()); + Object name = getParams().get("name"); + if (name!=null) { + return name.toString(); + } + return super.getName(); } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java index 8fbc617bd..e127a371e 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java @@ -17,6 +17,9 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; +import java.util.List; +import java.util.Map; + /** * A StmtsDef contains a list of rawStmts, as well as all of the optional * block parameters that can be assigned to {@link BlockParams}, which includes @@ -24,4 +27,18 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; */ public class RawStmtsBlock extends StatementsOwner { + public RawStmtsBlock() {} + + public void setFieldsByReflection(Map propsmap) { +// Object stmtValues = propsmap.remove("statements"); +// if (stmtValues!=null) { +// if (stmtValues instanceof List) { +// setByObject(stmtValues); +// } else { +// throw new RuntimeException("Invalid type for statements property: " + stmtValues.getClass().getCanonicalName()); +// } +// } + super.setFieldsByReflection(propsmap); + } + } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java index c408053b9..56548161a 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java @@ -19,6 +19,8 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** * A statements doc can have both a list of statement blocks and/or a @@ -35,7 +37,39 @@ public class RawStmtsDoc extends StatementsOwner { private List blocks = new ArrayList<>(); // no-args ctor is required - public RawStmtsDoc() {} + public RawStmtsDoc() { + } + + public void setFieldsByReflection(Map properties) { + Object blocksObjects = properties.remove("blocks"); + if (blocksObjects instanceof List) { + List blockList = ((List) blocksObjects); + for (Object blockData : blockList) { + if (blockData instanceof Map) { + Map blockDataMap = (Map)blockData; + RawStmtsBlock rawStmtsBlock = new RawStmtsBlock(); + rawStmtsBlock.setFieldsByReflection(blockDataMap); + blocks.add(rawStmtsBlock); + } else { + throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName()); + } + } + } + + Object scenariosData = properties.remove("scenarios"); + if (scenariosData!=null) { + if (scenariosData instanceof Map) { + RawScenarios rawScenarios = new RawScenarios(); + Map scenariosObjMap = (Map)scenariosData; + rawScenarios.setPropertiesByReflection(scenariosObjMap); + } else { + throw new RuntimeException("Invalid type for scenarios data: " + scenariosData.getClass().getCanonicalName()); + } + } + + super.setFieldsByReflection(properties); + + } /** * Return the list of statement blocks in this RawStmtsDoc. @@ -69,4 +103,5 @@ public class RawStmtsDoc extends StatementsOwner { public void setScenarios(RawScenarios scenarios) { this.scenarios = scenarios; } + } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java index bd3d35853..ef67ba84c 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java @@ -89,6 +89,7 @@ public class RawYamlStatementLoader { try { Iterable objects = yaml.loadAll(data); + List stmtListList = new ArrayList<>(); for (Object object : objects) { RawStmtsDoc tgsd = (RawStmtsDoc) object; diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java index 7e335a205..6b47618e4 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java @@ -34,6 +34,21 @@ public class StatementsOwner extends BlockParams { this.rawStmtDefs = rawStmtDefs; } + public void setFieldsByReflection(Map propsmap) { + Object statementsObject = propsmap.remove("statements"); + if (statementsObject!=null) { + setByObject(statementsObject); + } +// if (statementsObject!=null) { +// if (statementsObject instanceof List) { +// setByObject(statementsObject); +// } else { +// throw new RuntimeException("Invalid type for statements property: " + statementsObject.getClass().getCanonicalName()); +// } +// } + super.setFieldsByReflection(propsmap); + } + @SuppressWarnings("unchecked") public void setByObject(Object object) { if (object instanceof List) { @@ -61,7 +76,7 @@ public class StatementsOwner extends BlockParams { valueMap.put("name", entries.getKey()); itemizedMaps.add(valueMap); } else if (value instanceof String) { - Map stmtDetails = new HashMap() {{ + Map stmtDetails = new HashMap<>() {{ put("name", entries.getKey()); put("stmt", entries.getValue()); }}; diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/Tags.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/Tags.java index 2b89e235d..c240c81e2 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/Tags.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/Tags.java @@ -25,7 +25,7 @@ import java.util.Map; public class Tags implements Tagged { - private Map tags = new LinkedHashMap<>(); + private final Map tags = new LinkedHashMap<>(); @Override public Map getTags() { @@ -37,4 +37,17 @@ public class Tags implements Tagged { this.tags.putAll(tags); } + @SuppressWarnings("unchecked") + public void setFieldsByReflection(Map propsmap) { + Object tagsValues = propsmap.remove("tags"); + if (tagsValues != null) { + if (tagsValues instanceof Map) { + Map tagsMap = (Map) tagsValues; + tagsMap.forEach((ko, vo) -> tags.put(ko.toString(), vo.toString())); + } else { + throw new RuntimeException("Invalid type for tags property: " + tags.getClass().getCanonicalName()); + } + } + } + } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/stmtloader/GenericStmtLoader.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/stmtloader/GenericStmtLoader.java new file mode 100644 index 000000000..7f51c5b00 --- /dev/null +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/stmtloader/GenericStmtLoader.java @@ -0,0 +1,87 @@ +package io.nosqlbench.engine.api.activityconfig.stmtloader; + +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList; +import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; +import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError; +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.content.NBIO; +import io.nosqlbench.nb.api.errors.BasicError; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.DiscreteProbabilityBuffer; +import org.slf4j.Logger; +import org.yaml.snakeyaml.Yaml; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.StreamSupport; + +public class GenericStmtLoader { + + List> stringTransformers = new ArrayList<>(); + private final ArrayList> transformers = new ArrayList<>(); + + public void addTransformers(Function... newTransformers) { + Collections.addAll(this.transformers, newTransformers); + } + + public RawStmtsDocList load( + Logger logger, + String path, + String... searchPaths) { + + String data = null; + try { + Optional> oyaml = NBIO.all().prefix(searchPaths).name(path).extension("yaml").first(); + data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path)); + } catch (Exception e) { + throw new RuntimeException("error while reading file " + path, e); + } + + try { + if (logger != null) logger.debug("Applying string transformer to yaml data:" + data); + for (Function transformer : transformers) { + data = transformer.apply(data); + } + } catch (Exception e) { + RuntimeException t = new ActivityInitializationError("Error applying string applyTransforms to input", e); + if (logger != null) logger.error(t.getMessage(), t); + throw t; + } + + return parseYaml(logger, data); + + } + + private RawStmtsDocList parseYaml(Logger logger, String data) { + Yaml yaml = new Yaml(); + Iterable objects = yaml.loadAll(data); + List newDocList = new ArrayList<>(); + + for (Object object : objects) { + if (object instanceof Map) { + RawStmtsDoc doc = new RawStmtsDoc(); + doc.setFieldsByReflection((Map) object); + newDocList.add(doc); + } else { + throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName()); + } + } + RawStmtsDocList rawStmtsDocList = new RawStmtsDocList(newDocList); + return rawStmtsDocList; + } + + protected String applyTransforms(Logger logger, String data) { + for (Function xform : stringTransformers) { + try { + if (logger != null) logger.debug("Applying string transformer to yaml data:" + xform); + data = xform.apply(data); + } catch (Exception e) { + RuntimeException t = new ActivityInitializationError("Error applying string applyTransforms to input", e); + if (logger != null) logger.error(t.getMessage(), t); + throw t; + } + } + return data; + } + +} diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDef.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDef.java index e6f5ee64e..f82665478 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDef.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDef.java @@ -22,7 +22,10 @@ import io.nosqlbench.engine.api.activityconfig.ParsedStmt; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef; import io.nosqlbench.engine.api.util.Tagged; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; public class StmtDef implements Tagged { @@ -43,15 +46,58 @@ public class StmtDef implements Tagged { } public Map getBindings() { - return new MultiMapLookup(rawStmtDef.getBindings(), block.getBindings()); + return new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings()); } - public Map getParams() { - return new MultiMapLookup(rawStmtDef.getParams(), block.getParams()); + public Map getParams() { + return new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams()); } + public Map getParamsAsValueType(Class type) { + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams()); + Map map = new LinkedHashMap<>(); + lookup.forEach((k,v)->map.put(k,type.cast(v))); + return map; + } + + @SuppressWarnings("unchecked") + public V getParamOrDefault(String name, V defaultValue) { + Objects.requireNonNull(defaultValue); + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams()); + + if (!lookup.containsKey(name)) { + return defaultValue; + } + Object value = lookup.get(name); + return (V) defaultValue.getClass().cast(value); + } + + public V getParam(String name, Class type) { + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams()); + Object object = lookup.get(name); + V value = type.cast(object); + return value; + } + + @SuppressWarnings("unchecked") + public Optional getOptionalParam(String name, Class type) { + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams()); + if (lookup.containsKey(name)) { + Object object = lookup.get(name); + if (object==null) { + return Optional.empty(); + } + return Optional.of((V) type.cast(object)); + } + return Optional.empty(); + } + + public Optional getOptionalParam(String name) { + return getOptionalParam(name,String.class); + } + public Map getTags() { - return new MultiMapLookup(rawStmtDef.getTags(), block.getTags()); + return new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags()); } @Override diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java index aa15be47d..fb12eea99 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java @@ -21,11 +21,9 @@ import io.nosqlbench.engine.api.activityconfig.MultiMapLookup; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock; import io.nosqlbench.engine.api.util.Tagged; +import org.jetbrains.annotations.NotNull; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; public class StmtsBlock implements Tagged, Iterable { @@ -43,13 +41,13 @@ public class StmtsBlock implements Tagged, Iterable { } public List getStmts() { - + List rawStmtDefs = new ArrayList<>(); List statements = rawStmtsBlock.getRawStmtDefs(); for (int i = 0; i < statements.size(); i++) { rawStmtDefs.add( - new StmtDef(this,statements.get(i)) + new StmtDef(this, statements.get(i)) ); } return rawStmtDefs; @@ -69,18 +67,44 @@ public class StmtsBlock implements Tagged, Iterable { } public Map getTags() { - return new MultiMapLookup(rawStmtsBlock.getTags(), rawStmtsDoc.getTags()); + return new MultiMapLookup<>(rawStmtsBlock.getTags(), rawStmtsDoc.getTags()); } - public Map getParams() { - return new MultiMapLookup(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + public Map getParams() { + return new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + } + + public Map getParamsAsText() { + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + LinkedHashMap stringmap = new LinkedHashMap<>(); + lookup.forEach((k, v) -> stringmap.put(k, v.toString())); + return stringmap; + } + + @SuppressWarnings("unchecked") + public V getParamOrDefault(String name, V defaultValue) { + Objects.requireNonNull(defaultValue); + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + if (!lookup.containsKey(name)) { + return defaultValue; + } + Object value = lookup.get(name); + return (V) defaultValue.getClass().cast(value); + } + + public V getParam(String name, Class type) { + MultiMapLookup lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams()); + Object object = lookup.get(name); + V value = type.cast(object); + return value; } public Map getBindings() { - return new MultiMapLookup(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings()); + return new MultiMapLookup<>(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings()); } @Override + @NotNull public Iterator iterator() { return getStmts().iterator(); } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java index f4cb23629..24ba24e08 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java @@ -68,7 +68,7 @@ public class StmtsDoc implements Tagged, Iterable { /** * @return a usable map of parameters, including those inherited from the parent doc */ - public Map getParams() { + public Map getParams() { return rawStmtsDoc.getParams(); } diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDefDefsTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDefDefsTest.java index 38c31c422..1394eac1d 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDefDefsTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDefDefsTest.java @@ -62,6 +62,39 @@ public class RawStmtDefDefsTest { assertThat(block22.getTags().get("root1")).isEqualTo("value23"); } + @Test + public void testLayeringWithGenericLoader() { + + StmtsDocList all = StatementsLoader.load( + StatementsLoader.Loader.generified, + logger, + "testdocs" + "/docs_blocks_stmts.yaml" + ); + assertThat(all).isNotNull(); + assertThat(all.getStmtDocs()).hasSize(2); + StmtsDoc doc1 = all.getStmtDocs().get(0); + assertThat(doc1.getName()).isEqualTo("doc1"); + assertThat(doc1.getBlocks()).hasSize(1); + StmtsDoc doc2 = all.getStmtDocs().get(1); + assertThat(doc2.getBlocks()).hasSize(2); + + StmtsBlock block1 = doc1.getBlocks().get(0); + assertThat(block1.getBindings()).hasSize(2); + assertThat(block1.getName()).isEqualTo("doc1--block0"); + assertThat(block1.getTags()).hasSize(1); + + StmtsBlock block21 = doc2.getBlocks().get(0); + StmtsBlock block22 = doc2.getBlocks().get(1); + + assertThat(block21.getName()).isEqualTo("doc2--block1"); + assertThat(block21.getTags()).hasSize(3); + + assertThat(block22.getName()).isEqualTo("doc2--block2"); + assertThat(block22.getTags()).hasSize(2); + assertThat(block22.getTags().get("root1")).isEqualTo("value23"); + } + + @Test public void testStatementRendering() { StmtsDocList all = StatementsLoader.load(logger, "testdocs/docs_blocks_stmts.yaml"); @@ -78,4 +111,26 @@ public class RawStmtDefDefsTest { } + @Test + public void testStatementRenderingGenerified() { + StmtsDocList all = StatementsLoader.load( + StatementsLoader.Loader.generified, + logger, + "testdocs" + + "/docs_blocks_stmts.yaml" + ); + + assertThat(all).isNotNull(); + assertThat(all.getStmtDocs()).hasSize(2); + StmtsDoc doc1 = all.getStmtDocs().get(0); + StmtsBlock block1 = doc1.getBlocks().get(0); + assertThat(block1.getName()).isEqualTo("doc1--block0"); + List assys = block1.getStmts(); + assertThat(assys).hasSize(2); + StmtDef sdef1 = assys.get(0); + assertThat(sdef1.getName()).isEqualTo("doc1--block0--stmt1"); + assertThat(assys.get(0).getStmt()).isEqualTo("s1"); + + } + } diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java index 553cb27ba..ac0847a35 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java @@ -101,7 +101,7 @@ public class StmtsDocListTest { StmtsBlock block0 = doc1.getBlocks().get(0); Map doc1block0tags = block0.getTags(); - Map doc1block0params = block0.getParams(); + Map doc1block0params = block0.getParamsAsText(); Map doc1block0bindings = block0.getBindings(); assertThat(doc1block0tags).hasSize(3);