IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1714 from nosqlbench/jshook/talk-improvements

Browse Source 
Jshook/talk improvements
This commit is contained in:
Jonathan Shook 2023-12-15 11:12:19 -08:00 committed by GitHub
commit 84bc7bd4fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
174 changed files with 858 additions and 906 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpDriverAdapter.java
View File

@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-amqp/src/main/java/io/nosqlbench/adapter/amqp/AmqpDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.amqp;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "amqp")

32
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/CqlDriverAdapterLoader.java Normal file
View File

@ -0,0 +1,32 @@
package io.nosqlbench.adapter.cqld4;
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.labels.NBLabels;
@Service(value = DriverAdapterLoader.class, selector = "cql")
public class CqlDriverAdapterLoader implements DriverAdapterLoader {
@Override
public Cqld4DriverAdapter load(NBComponent parent, NBLabels childLabels) {
return new Cqld4DriverAdapter(parent, childLabels);
}
}

2
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/CqlDriverAdapterStub.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.adapter.cqld4;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapter.class, selector = "cql")

2
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapter.java
View File

@ -25,7 +25,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4DriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.cqld4;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "cqld4")

19
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java
View File

@ -99,7 +99,7 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
this.metrics = metrics;
}
public final List<Row> apply(long cycle) {
public final ArrayList<Row> apply(long cycle) {
Statement<?> statement = getStmt();
logger.trace(() -> "apply() invoked, statement obtained, executing async with page size: " + statement.getPageSize() + " thread local rows: ");
@ -118,7 +118,7 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
});
try {
return rowsStage.toCompletableFuture().get(300, TimeUnit.SECONDS);
return new PrintableRowList(rowsStage.toCompletableFuture().get(300, TimeUnit.SECONDS));
} catch (ExecutionException exe) {
Throwable ee = exe.getCause();
if (ee instanceof RuntimeException re) {
@ -142,6 +142,21 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
// processors.flush();
}
private static class PrintableRowList extends ArrayList<Row> {
public PrintableRowList(List<Row> values) {
super(values);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (Row row : this) {
sb.append(row.getFormattedContents());
}
return sb.toString();
}
}
// private BiFunction<AsyncResultSet,Throwable> handler
@Override
public Op getNextOp() {

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.nb.api.components.NBNamedElement;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import io.nosqlbench.cqlgen.model.CqlModel;
import java.util.function.Function;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnBase.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.cqlgen.model;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBNamedElement;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
public abstract class CqlColumnBase implements NBNamedElement, NBLabeledElement {

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspaceDef.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.cqlgen.model;
import com.datastax.oss.driver.internal.core.util.Strings;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBNamedElement;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.cqlgen.model;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBNamedElement;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.cqlgen.core.CGTableStats;
import io.nosqlbench.cqlgen.transformers.ComputedTableStats;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.cqlgen.model;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBNamedElement;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import java.util.ArrayList;
import java.util.List;

4
adapter-cqld4/src/main/java/io/nosqlbench/engine/extensions/vectormath/CqlUtils.java
View File

@ -17,8 +17,8 @@
package io.nosqlbench.engine.extensions.vectormath;
import com.datastax.oss.driver.api.core.cql.Row;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.util.List;
import java.util.Objects;

2
adapter-cqld4/src/main/java/io/nosqlbench/engine/extensions/vectormath/CqlUtilsPluginInfo.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.extensions.vectormath;
import io.nosqlbench.nb.api.extensions.ScriptingExtensionPluginInfo;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.Logger;

12
adapter-cqld4/src/main/resources/cql.md Normal file
View File

@ -0,0 +1,12 @@
# cql
In this version of NoSQLBench, the CQL driver is fully
based on version 4 of the
[Java Driver for Apache Cassandra](https://github.com/apache/cassandra-java-driver)
version 4.X.
This `cqld4` driver has replaced the older CQL driver under the name `cql`.
Previous versions of the CQL drivers are no longer bundled with NB5. Instead,
when you specify either `cql` or `cqld4`, the version 4 drivers is used.
See the help for `cqld4` for more details.

2
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java
View File

@ -26,7 +26,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.params.NBParams;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;

2
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapterLoader.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.adapter.diag;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "diag")

2
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagOpDispenser.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.NBReconfigurable;
import io.nosqlbench.nb.api.components.NBParentComponentInjection;
import io.nosqlbench.nb.api.components.core.NBParentComponentInjection;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.nb.annotations.ServiceSelector;
import org.apache.logging.log4j.LogManager;

2
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/optasks/BaseDiagTask.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.adapter.diag.optasks;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.util.Map;

2
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/optasks/DiagTask.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.adapter.diag.optasks;
import io.nosqlbench.nb.api.config.standard.NBConfigurable;
import io.nosqlbench.nb.api.config.standard.NBReconfigurable;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.components.NBParentComponentInjection;
import io.nosqlbench.nb.api.components.core.NBParentComponentInjection;
import java.util.Map;
import java.util.function.BiFunction;

2
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/optasks/DiagTask_gauge.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.Param;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBParentComponentInjection;
import io.nosqlbench.nb.api.components.core.NBParentComponentInjection;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import io.nosqlbench.virtdata.core.bindings.DataMapper;

2
adapter-diag/src/test/java/io/nosqlbench/adapter/diag/optasks/DiagTask_gaugeTest.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.adapter.diag.optasks;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.TestComponent;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import org.assertj.core.data.Offset;
import org.junit.jupiter.api.Test;

2
adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBDriverAdapter.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;

2
adapter-dynamodb/src/main/java/io/nosqlbench/adapter/dynamodb/DynamoDBDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.dynamodb;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "dynamodb")

6
adapter-http/pom.xml
View File

@ -57,6 +57,12 @@
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator</artifactId>
<version>7.1.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.swagger.parser.v3</groupId>

2
adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapter.java
View File

@ -27,7 +27,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;

2
adapter-http/src/main/java/io/nosqlbench/adapter/http/HttpDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.http;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "http")

2
adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpMetrics.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.adapter.http.core;
import com.codahale.metrics.Histogram;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
public class HttpMetrics implements NBLabeledElement {
private final NBComponent parent;

2
adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.Param;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/JDBCDriverAdapter.java
View File

@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/utils/JDBCDriverAdapterLoader.java
View File

@ -21,7 +21,7 @@ package io.nosqlbench.adapter.jdbc.utils;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.adapter.jdbc.JDBCDriverAdapter;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "jdbc")

2
adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaDriverAdapter.java
View File

@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/KafkaDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.kafka;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "kafka")

2
adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.mongodb.core;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "mongodb")

2
adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoSpace.java
View File

@ -22,7 +22,7 @@ import com.mongodb.ServerApi;
import com.mongodb.ServerApiVersion;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import io.nosqlbench.nb.api.components.NBNamedElement;
import io.nosqlbench.nb.api.components.core.NBNamedElement;
import com.mongodb.client.MongoDatabase;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;

2
adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbDriverAdapter.java
View File

@ -23,7 +23,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import java.util.function.Function;

2
adapter-pinecone/src/main/java/io/nosqlbench/adapter/pinecone/PineconeDriverAdapter.java
View File

@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import java.util.function.Function;

2
adapter-pinecone/src/main/java/io/nosqlbench/adapter/pinecone/PineconeDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.pinecone;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "pinecone")

4
adapter-pinecone/src/main/java/io/nosqlbench/engine/extensions/vectormath/PineconeScriptingUtils.java
View File

@ -16,8 +16,8 @@
package io.nosqlbench.engine.extensions.vectormath;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.pinecone.proto.QueryResponse;
import io.pinecone.proto.ScoredVector;

2
adapter-pinecone/src/main/java/io/nosqlbench/engine/extensions/vectormath/PineconeScriptingUtilsPluginInfo.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.extensions.vectormath;
import io.nosqlbench.nb.api.extensions.ScriptingExtensionPluginInfo;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.Logger;

2
adapter-pinecone/src/test/java/io/nosqlbench/engine/extensions/vectormath/PineconeScriptingUtilsTest.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.engine.extensions.vectormath;
import com.google.protobuf.Struct;
import com.google.protobuf.Value;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.pinecone.proto.QueryResponse;
import io.pinecone.proto.ScoredVector;
import org.junit.jupiter.api.Test;

2
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapter.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;

2
adapter-pulsar/src/main/java/io/nosqlbench/adapter/pulsar/PulsarDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.pulsar;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "pulsar")

2
adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapter.java
View File

@ -24,7 +24,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-s4j/src/main/java/io/nosqlbench/adapter/s4j/S4JDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.s4j;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "s4j")

2
adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java
View File

@ -25,7 +25,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;

2
adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.stdout;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "stdout")

2
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java
View File

@ -28,7 +28,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.tcpclient;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "tcpclient")

2
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java
View File

@ -27,7 +27,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapterLoader.java
View File

@ -20,7 +20,7 @@ package io.nosqlbench.adapter.tcpserver;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Service;
@Service(value = DriverAdapterLoader.class, selector = "tcpserver")

2
adapters-api/src/main/java/io/nosqlbench/adapter/diag/DriverAdapterLoader.java
View File

@ -21,7 +21,7 @@ package io.nosqlbench.adapter.diag;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
public interface DriverAdapterLoader {
public <A extends Op,B> DriverAdapter<A,B> load(NBComponent parent, NBLabels childLabels);

24
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java
View File

@ -17,6 +17,7 @@
package io.nosqlbench.adapters.api.activityconfig;
import com.amazonaws.util.StringInputStream;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
@ -27,6 +28,8 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.adapters.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import scala.Option;
import sjsonnet.DefaultParseCache;
import sjsonnet.SjsonnetMain;
@ -146,4 +149,25 @@ public class OpsLoader {
return stdoutOutput;
}
// TODO These should not be exception based, use explicit pattern checks instead, or tap
// into the parsers in a non-exception way
public static boolean isJson(String workload) {
try {
new GsonBuilder().setPrettyPrinting().create().fromJson(workload, Map.class);
return true;
} catch (Exception e) {
return false;
}
}
// TODO These should not be exception based, use explicit pattern checks instead, or tap
// into the parsers in a non-exception way
public static boolean isYaml(String workload) {
try {
Object result = new Load(LoadSettings.builder().build()).loadFromString(workload);
return (result instanceof Map);
} catch (Exception e) {
return false;
}
}
}

2
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/OpsOwner.java
View File

@ -22,7 +22,7 @@ import java.util.*;
public class OpsOwner extends RawOpFields {
private final static List<String> opsFieldNames = List.of("op","ops","operation","statement","statements");
private final static List<String> opsFieldNames = List.of("op","ops","operation","stmt","statement","statements");
private List<RawOpDef> rawOpDefs = new ArrayList<>();

18
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -16,6 +16,8 @@
package io.nosqlbench.adapters.api.activityconfig.rawyaml;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
@ -36,6 +38,11 @@ public class RawOpsLoader {
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
private static LoadSettings loadSettings = LoadSettings.builder().build();
private final Load yaml = new Load(loadSettings);
private final Gson gson = new GsonBuilder().setPrettyPrinting().create();
public RawOpsLoader(Function<String,String> transformer) {
addTransformer(transformer);
}
@ -44,6 +51,15 @@ public class RawOpsLoader {
addTransformer(new StrInterpolator());
}
public boolean isJson(String workload) {
try {
Object canLoad = gson.fromJson(workload, Object.class);
} catch (Exception e) {
return false;
}
return true;
}
private void addTransformer(Function<String, String> newTransformer) {
Collections.addAll(this.transformers, newTransformer);
}
@ -79,8 +95,6 @@ public class RawOpsLoader {
}
public RawOpsDocList parseYaml(String data) {
LoadSettings loadSettings = LoadSettings.builder().build();
Load yaml = new Load(loadSettings);
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawOpsDoc> newDocList = new ArrayList<>();

3
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java
View File

@ -21,12 +21,11 @@ import groovy.lang.Binding;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.adapters.api.evalctx.*;
import io.nosqlbench.adapters.api.evalctx.*;
import io.nosqlbench.adapters.api.metrics.ThreadLocalNamedTimers;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/docs/BundledDriverAdapterDocs.java
View File

@ -24,7 +24,7 @@ import io.nosqlbench.nb.api.docsapi.Docs;
import io.nosqlbench.nb.api.docsapi.DocsBinder;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.spi.SimpleServiceLoader;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;

6
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java
View File

@ -21,8 +21,8 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.fieldmappers.FieldDestruc
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.config.standard.*;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -176,7 +176,7 @@ public abstract class BaseDriverAdapter<R extends Op, S> extends NBBaseComponent
.add(Param.optional("instrument", Boolean.class))
.add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
.add(Param.optional("driver", String.class))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|none)"))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
.add(Param.optional("maxtries", Integer.class))
.asReadOnly();
}

2
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.nb.api.docsapi.Docs;
import io.nosqlbench.nb.api.docsapi.DocsBinder;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;

34
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/EmitterOp.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapters.api.activityimpl.uniform;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
public class EmitterOp implements CycleOp<Object> {
private final CycleOp<?> cycleOp;
public EmitterOp(CycleOp<?> cycleOp) {
this.cycleOp = cycleOp;
}
@Override
public Object apply(long value) {
Object result = cycleOp.apply(value);
System.out.println("result from cycle " + value + ":\n"+result);
return result;
}
}

38
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/EmitterOpDispenserWrapper.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapters.api.activityimpl.uniform;
import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.adapters.api.templating.ParsedOp;
public class EmitterOpDispenserWrapper extends BaseOpDispenser<Op, Object> {
private final OpDispenser<? extends CycleOp<?>> realDispenser;
public EmitterOpDispenserWrapper(DriverAdapter<Op,Object> adapter, ParsedOp pop, OpDispenser<? extends CycleOp<?>> realDispenser) {
super(adapter, pop);
this.realDispenser = realDispenser;
}
@Override
public EmitterOp apply(long cycle) {
CycleOp<?> cycleOp = realDispenser.apply(cycle);
return new EmitterOp(cycleOp);
}
}

342
adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/CommandTemplate.java
View File

@ -1,342 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapters.api.templating;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.nb.api.config.params.ParamsParser;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.*;
import java.util.function.Function;
/**
* This is a general purpose template which uses a map of named parameters.
* The result is a template which is comprised of a map of names and values, which can
* be used to create a cycle-specific map of values that can describe a literal operation
* for some native driver. How this map is used is context dependent.
*
* Generally speaking, the properties in this map are taken as parameters or field values,
* or a command verb. How the keys in the resulting map are used to construct an operation
* for execution is entirely dependent on how a developer wants to map these fields to
* a native driver's API.
*
* A CommandTemplate can be crated directly, or from an OpTemplate. Additional map parsers
* may be provided when needed for specialized forms of syntax or variations which should also
* be supported. See the constructor docs for details on these variations.
*/
@Deprecated(forRemoval = true)
public class CommandTemplate {
private final static Logger logger = LogManager.getLogger(CommandTemplate.class);
private final String name;
private final Map<String, String> statics = new HashMap<>();
private final Map<String, StringBindings> dynamics = new HashMap<>();
transient private final int mapsize;
/**
* Create a CommandTemplate directly from an OpTemplate.
*
* In this form, if {@link OpTemplate#getOp()}
* is non-null, then it taken as a line-oriented value and parsed according to default {@link ParamsParser} behavior.
*
* Additionally, any op params provided are considered as entries to add to the command template's map.
*
* @param optpl An OpTemplate
*/
public CommandTemplate(OpTemplate optpl) {
this(optpl.getName(), optpl.getOp().toString(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), List.of());
}
/**
* Create a CommandTemplate directly from an OpTemplate, as in {@link #CommandTemplate(OpTemplate)},
* with added support for parsing the oneline form with the provided parsers.
*
* In this form, if {@link OpTemplate#getOp()}
* is non-null, then it taken as a line-oriented value and parsed according to default {@link ParamsParser} behavior.
* However, the provided parsers (if any) are used first in order to match alternate forms of syntax.
*
* See {@link CommandTemplate#CommandTemplate(String, String, Map, Map, List)} for full details on the provided
* parsers.
*
* @param optpl An OpTemplate
* @param parsers A list of parser functions
*/
public CommandTemplate(OpTemplate optpl, List<Function<String, Map<String, String>>> parsers) {
this(optpl.getName(), optpl.getStmt().orElseThrow(), optpl.getParamsAsValueType(String.class), optpl.getBindings(), parsers);
}
/**
* Create a command template from a set of optional properties.
*
* <P>The parsers provided should honor these expectations:
* <UL>
* <LI>If the one-line format is not recognized, the parser should return null.</LI>
* <LI>If the one-line format is recognized, and the values provided are valid, then they should be
* returned as a {@link Map} of {@link String} to {@link String}.</LI>
* <LI>Otherwise the parser should throw an exception, signifying either an internal parser error or
* invalid data.</LI>
* </UL>
*
* If none of the provided parsers (if any) return a map of values for the one-line format, then the default
* behavior of {@link ParamsParser} is used.
* </P>
*
* @param name The name of the command template
* @param op An object version of the parameters to be parsed by {@link ParamsParser}
* @param params A set of named parameters and values in name:value form.
* @param bindings A set of named bindings in name:recipe form.
* @param optionalParsers A set of functions which, if provided, will be used to read the oneline form.
*/
public CommandTemplate(
String name,
String op,
Map<String, String> params,
Map<String, String> bindings,
List<Function<String, Map<String, String>>> optionalParsers
) {
this.name = name;
Map<String, String> cmd = new HashMap<>();
// Only parse and inject the one-line form if it is defined.
// The first parser to match and return a map will be the last one tried.
// If none of the supplemental parsers work, the default params parser is used
String oneline;
if (op instanceof CharSequence) {
oneline = op;
} else {
throw new BasicError("Unable to create a oneline version of the CommandTemplate with op type of " + op.getClass().getSimpleName());
}
if (oneline != null) {
List<Function<String, Map<String, String>>> parserlist = new ArrayList<>(optionalParsers);
parserlist.add(s -> ParamsParser.parse(s, false));
boolean didParse = false;
for (Function<String, Map<String, String>> parser : parserlist) {
Map<String, String> parsed = parser.apply(oneline);
if (parsed != null) {
logger.debug(() -> "parsed request: " + parsed);
cmd.putAll(parsed);
didParse = true;
break;
}
}
if (!didParse) {
throw new RuntimeException("A oneline form was provided for the command template, but none of the " +
"provided" +
" parsers were able to parse it, not even ParamsParser.parse(...)");
}
}
// Always add the named params, but warn if they overwrite any oneline named params
params.forEach((k, v) -> {
if (cmd.containsKey(k)) {
logger.warn("command property override: '" + k + "' superseded by param form with value '" + v + "'");
}
});
cmd.putAll(params);
cmd.forEach((param, value) -> {
ParsedTemplateString paramTemplate = new ParsedTemplateString(value, bindings);
if (paramTemplate.getBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();
dynamics.put(param, paramStringBindings);
} else {
statics.put(param, value);
}
});
this.mapsize = statics.size() + dynamics.size();
}
/**
* Apply the provided binding functions to the command template, yielding a map with concrete values
* to be used by a native command.
*
* @param cycle The cycle value which will be used by the binding functions
* @return A map of specific values
*/
public Map<String, String> getCommand(long cycle) {
HashMap<String, String> map = new HashMap<>(mapsize);
map.putAll(statics);
dynamics.forEach((k, v) -> {
map.put(k, v.bind(cycle));
});
return map;
}
/**
* The name of the operation
*/
public String getName() {
return name;
}
/**
* True if the command template contains all static (non-binding) values.
*/
public boolean isStatic() {
return this.dynamics.size() == 0;
}
public boolean isStatic(String keyname) {
return this.statics.containsKey(keyname);
}
public boolean isStaticSet(String... keynames) {
for (String keyname : keynames) {
if (!isStatic(keyname)) {
return false;
}
}
return true;
}
public boolean isDynamicSet(String... keynames) {
for (String keyname : keynames) {
if (!isDynamic(keyname)) {
return false;
}
}
return true;
}
public boolean isDynamic(String keyname) {
return this.dynamics.containsKey(keyname);
}
public boolean containsKey(String keyname) {
return this.statics.containsKey(keyname) || this.dynamics.containsKey(keyname);
}
/**
* The set of key names known by this command template.
*/
public Set<String> getPropertyNames() {
return this.statics.keySet();
}
@Override
public String toString() {
return "CommandTemplate{" +
"name='" + name + '\'' +
", statics=" + statics +
", dynamics=" + dynamics +
'}';
}
public String getStatic(String staticVar) {
return statics.get(staticVar);
}
public String getDynamic(String dynamicVar, long input) {
return dynamics.get(dynamicVar).bind(input);
}
public String get(String var, long input) {
if (statics.containsKey(var)) {
return statics.get(var);
}
if (dynamics.containsKey(var)) {
return dynamics.get(var).bind(input);
}
return null;
}
public String getOr(String var, long input, String defaultVal) {
if (statics.containsKey(var)) {
return statics.get(var);
}
if (dynamics.containsKey(var)) {
return dynamics.get(var).bind(input);
}
return defaultVal;
}
public String getStaticOr(String staticVar, String defaultVal) {
if (statics.containsKey(staticVar)) {
return statics.get(staticVar);
}
return defaultVal;
}
public String getDynamicOr(String dynamicVar, long input, String defaultVal) {
if (dynamics.containsKey(dynamicVar)) {
return getDynamic(dynamicVar, input);
} else {
return defaultVal;
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CommandTemplate that = (CommandTemplate) o;
return Objects.equals(name, that.name) && Objects.equals(statics, that.statics) && Objects.equals(dynamics, that.dynamics);
}
@Override
public int hashCode() {
return Objects.hash(name, statics, dynamics);
}
public boolean containsAny(String... varNames) {
for (String varName : varNames) {
if (this.containsKey(varName)) {
return true;
}
}
return false;
}
public boolean isStaticOrUnsetSet(String... varnames) {
for (String varname : varnames) {
if (isDynamic(varname)) {
return false;
}
}
return true;
}
/**
* This should only be used to provide a view of a field definition, never for actual use in a payload.
* @param varname The field name which you want to explain
* @return A string representation of the field name
*/
public String getFieldDescription(String varname) {
if (this.isDynamic(varname)) {
return "dynamic: " + this.dynamics.get(varname).toString();
} else if (this.isStatic(varname)) {
return "static: " + this.getStatic(varname);
} else {
return "UNDEFINED";
}
}
}

4
adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/ParsedOp.java
View File

@ -24,8 +24,8 @@ import io.nosqlbench.nb.api.config.fieldreaders.StaticFieldReader;
import io.nosqlbench.nb.api.config.standard.NBConfigError;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.errors.OpConfigError;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.engine.api.templating.ObjectCache;
import io.nosqlbench.engine.api.templating.ParsedTemplateMap;
import io.nosqlbench.engine.api.templating.TypeAndTarget;

2
adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/ParsedOpTest.java
View File

@ -25,7 +25,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.Param;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import org.junit.jupiter.api.Test;
import java.util.List;

2
devdocs/sketches/invokable_models.md
View File

@ -38,7 +38,7 @@ To clarify the differences, here are some basic examples:
jmxOp.execute();
kafkaSequencer.get(cycle).write(cycle);
mongoResultDoc = activity.getDatabase().runCommand(queryBson, rms.getReadPreference());
WebDriverVerbs.execute(cycle, commandTemplate, context, dryrun);
WebDriverVerbs.execute(cycle, cmdTemplate, context, dryrun);
ActivityType<A extends Activity>

35
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -34,7 +34,7 @@ import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.logging.NBLogLevel;
import io.nosqlbench.nb.api.metadata.SessionNamer;
import io.nosqlbench.nb.api.metadata.SystemId;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility;
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
import io.nosqlbench.engine.api.activityapi.input.InputType;
@ -42,7 +42,6 @@ import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.engine.cli.NBCLIOptions.Mode;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.lifecycle.ExecutionResult;
import io.nosqlbench.engine.core.clientload.ClientSystemMetricChecker;
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
import io.nosqlbench.engine.core.lifecycle.session.NBSession;
@ -273,6 +272,29 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
return NBCLI.EXIT_OK;
}
if (options.wantsToCatResource()) {
final String resourceToCat = options.wantsToCatResourceNamed();
NBCLI.logger.debug(() -> "user requests to cat " + resourceToCat);
Optional<Content<?>> tocat = NBIO.classpath()
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCat).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocat.isEmpty()) tocat = NBIO.classpath()
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCat).first();
final Content<?> data = tocat.orElseThrow(
() -> new BasicError("Unable to find " + resourceToCat +
" in classpath to cat out"));
System.out.println(data.get());
NBCLI.logger.info(() -> "Dumped internal resource '" + data.asPath() + "' to stdout");
return NBCLI.EXIT_OK;
}
if (options.wantsToCopyResource()) {
final String resourceToCopy = options.wantsToCopyResourceNamed();
NBCLI.logger.debug(() -> "user requests to copy out " + resourceToCopy);
@ -391,6 +413,12 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
),
sessionName
);
// TODO: Decide whether this should be part of ctor consistency
Map.of(
"summary", options.getReportSummaryTo(),
"logsdir", options.getLogsDirectory().toString(),
"progress", options.getProgressSpec()
).forEach(session::setComponentProp);
options.wantsReportCsvTo().ifPresent(cfg -> {
MetricInstanceFilter filter = new MetricInstanceFilter();
@ -417,7 +445,8 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
ExecutionResult sessionResult = session.apply(options.getCommands());
sessionResult.printSummary(System.out);
// sessionResult.printSummary(System.out);
logger.info(sessionResult);
return sessionResult.getStatus().code;
}

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIArgsFile.java
View File

@ -496,7 +496,7 @@ public class NBCLIArgsFile {
}
String opt = iter.next();
if (!opt.startsWith("-") || reservedPredicate.test(opt)) {
if (!opt.startsWith("-")) {
throw new RuntimeException("Arguments following the --pin option must not" +
" be commands like '" + opt + "'");
}

15
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -84,6 +84,7 @@ public class NBCLIOptions {
private static final String COMPILE_SCRIPT = "--compile-script";
private static final String SCRIPT_FILE = "--script-file";
private static final String COPY = "--copy";
private static final String CAT = "--cat";
private static final String SHOW_STACKTRACES = "--show-stacktraces";
private static final String EXPERIMENTAL = "--experimental";
private static final String MATURITY = "--maturity";
@ -199,6 +200,7 @@ public class NBCLIOptions {
private boolean wantsConsoleMetrics = true;
private String annotateLabelSpec = "";
private String metricsLabelSpec = "";
private String wantsToCatResource ="";
public boolean wantsLoggedMetrics() {
return this.wantsConsoleMetrics;
@ -264,6 +266,10 @@ public class NBCLIOptions {
return this.labels;
}
public boolean wantsToCatResource() {
return this.wantsToCatResource!=null && !this.wantsToCatResource.isEmpty();
}
public enum Mode {
ParseGlobalsOnly,
ParseAllOptions
@ -629,6 +635,10 @@ public class NBCLIOptions {
arglist.removeFirst();
this.wantsToCopyWorkload = this.readWordOrThrow(arglist, "workload to copy");
break;
case NBCLIOptions.CAT:
arglist.removeFirst();
this.wantsToCatResource = this.readWordOrThrow(arglist, "workload to cat");
break;
default:
nonincludes.addLast(arglist.removeFirst());
}
@ -887,6 +897,11 @@ public class NBCLIOptions {
return this.wantsToCopyWorkload;
}
public String wantsToCatResourceNamed() {
return this.wantsToCatResource;
}
public boolean wantsWorkloadsList() {
return this.wantsWorkloadsList;
}

6
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/Activity.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityapi.core;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressCapable;
@ -64,9 +64,9 @@ public interface Activity extends Comparable<Activity>, ActivityDefObserver, Pro
*/
void closeAutoCloseables();
MotorDispenser getMotorDispenserDelegate();
MotorDispenser<?> getMotorDispenserDelegate();
void setMotorDispenserDelegate(MotorDispenser motorDispenser);
void setMotorDispenserDelegate(MotorDispenser<?> motorDispenser);
InputDispenser getInputDispenserDelegate();

2
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityType.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityapi.core;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityapi.input.InputDispenser;
import io.nosqlbench.engine.api.activityapi.output.OutputDispenser;

2
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/ErrorMetrics.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityapi.errorhandling;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.api.metrics.ExceptionCountMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionExpectedResultVerificationMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionHistoMetrics;

2
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/ratelimits/RateLimiters.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityapi.ratelimits;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRate;
import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
import org.apache.logging.log4j.LogManager;

4
engine-core/src/main/java/io/nosqlbench/engine/api/activityapi/ratelimits/simrate/SimRate.java
View File

@ -17,8 +17,8 @@
package io.nosqlbench.engine.api.activityapi.ratelimits.simrate;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

204
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -16,8 +16,10 @@
package io.nosqlbench.engine.api.activityimpl;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.adapters.api.activityimpl.uniform.EmitterOpDispenserWrapper;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.events.ParamChange;
import io.nosqlbench.engine.api.activityapi.core.*;
import io.nosqlbench.engine.api.activityapi.core.progress.ActivityMetricProgressMeter;
@ -32,7 +34,6 @@ import io.nosqlbench.engine.api.activityapi.ratelimits.simrate.SimRateSpec;
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.errors.OpConfigError;
@ -51,19 +52,16 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DryRunOpDispenserWrapper;
import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.adapters.api.templating.CommandTemplate;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.InputStream;
import java.io.PrintWriter;
import java.lang.reflect.AnnotatedType;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
* A default implementation of an Activity, suitable for building upon.
@ -73,7 +71,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
protected ActivityDef activityDef;
private final List<AutoCloseable> closeables = new ArrayList<>();
private MotorDispenser motorDispenser;
private MotorDispenser<?> motorDispenser;
private InputDispenser inputDispenser;
private ActionDispenser actionDispenser;
private OutputDispenser markerDispenser;
@ -92,7 +90,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
private final RunStateTally tally = new RunStateTally();
public SimpleActivity(NBComponent parent, ActivityDef activityDef) {
super(parent,NBLabels.forKV("activity",activityDef.getAlias()).and(activityDef.auxLabels()));
super(parent, NBLabels.forKV("activity", activityDef.getAlias()).and(activityDef.auxLabels()));
this.activityDef = activityDef;
if (activityDef.getAlias().equals(ActivityDef.DEFAULT_ALIAS)) {
Optional<String> workloadOpt = activityDef.getParams().getOptionalString(
@ -111,7 +109,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
}
public SimpleActivity(NBComponent parent, String activityDefString) {
this(parent,ActivityDef.parseActivityDef(activityDefString));
this(parent, ActivityDef.parseActivityDef(activityDefString));
}
@Override
@ -123,7 +121,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
if (null == this.errorHandler) {
errorHandler = new NBErrorHandler(
() -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
() -> getExceptionMetrics());
this::getExceptionMetrics);
}
return errorHandler;
}
@ -147,12 +145,12 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
}
@Override
public final MotorDispenser getMotorDispenserDelegate() {
public final MotorDispenser<?> getMotorDispenserDelegate() {
return motorDispenser;
}
@Override
public final void setMotorDispenserDelegate(MotorDispenser motorDispenser) {
public final void setMotorDispenserDelegate(MotorDispenser<?> motorDispenser) {
this.motorDispenser = motorDispenser;
}
@ -202,7 +200,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
}
public String toString() {
return (activityDef!=null ? activityDef.getAlias() : "unset_alias" ) + ':' + this.runState + ':' + this.tally ;
return (activityDef != null ? activityDef.getAlias() : "unset_alias") + ':' + this.runState + ':' + this.tally;
}
@Override
@ -318,6 +316,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
public void createOrUpdateStrideLimiter(SimRateSpec spec) {
strideLimiter = RateLimiters.createOrUpdate(this, strideLimiter, spec);
}
public void createOrUpdateCycleLimiter(SimRateSpec spec) {
cycleLimiter = RateLimiters.createOrUpdate(this, cycleLimiter, spec);
}
@ -411,49 +410,23 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
" more information.");
}
if (0 < this.activityDef.getCycleCount() && 0 == seq.getOps().size()) {
if (0 < this.activityDef.getCycleCount() && seq.getOps().isEmpty()) {
throw new BasicError("You have configured a zero-length sequence and non-zero cycles. Tt is not possible to continue with this activity.");
}
}
/**
* Given a function that can create an op of type <O> from a CommandTemplate, generate
* an indexed sequence of ready to call operations.
* <p>
* This method works almost exactly like the ,
* except that it uses the {@link CommandTemplate} semantics, which are more general and allow
* for map-based specification of operations with bindings in each field.
* <p>
* It is recommended to use the CommandTemplate form
* than the
*
* @param <O>
* @param opinit
* @param strict
* @return
*/
protected <O extends Op> OpSequence<OpDispenser<? extends O>> createOpSequenceFromCommands(
Function<CommandTemplate, OpDispenser<O>> opinit,
boolean strict
) {
Function<OpTemplate, CommandTemplate> f = CommandTemplate::new;
Function<OpTemplate, OpDispenser<? extends O>> opTemplateOFunction = f.andThen(opinit);
return createOpSequence(opTemplateOFunction, strict, Optional.empty());
}
protected <O extends Op> OpSequence<OpDispenser<? extends O>> createOpSourceFromParsedOps(
Map<String, DriverAdapter> adapterCache,
Map<String, OpMapper<Op>> mapperCache,
List<DriverAdapter> adapters,
// Map<String, DriverAdapter<?,?>> adapterCache,
// Map<String, OpMapper<? extends Op>> mapperCache,
List<DriverAdapter<?,?>> adapters,
List<ParsedOp> pops
) {
try {
List<Long> ratios = new ArrayList<>(pops.size());
for (int i = 0; i < pops.size(); i++) {
ParsedOp pop = pops.get(i);
for (ParsedOp pop : pops) {
long ratio = pop.takeStaticConfigOr("ratio", 1);
ratios.add(ratio);
}
@ -472,16 +445,21 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
logger.info(() -> "skipped mapping op '" + pop.getName() + '\'');
continue;
}
String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
boolean dryrun = "op".equalsIgnoreCase(dryrunSpec);
DriverAdapter adapter = adapters.get(i);
OpMapper opMapper = adapter.getOpMapper();
DriverAdapter<?,?> adapter = adapters.get(i);
OpMapper<? extends Op> opMapper = adapter.getOpMapper();
OpDispenser<? extends Op> dispenser = opMapper.apply(pop);
if (dryrun) {
dispenser = new DryRunOpDispenserWrapper(adapter, pop, dispenser);
String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
if ("op".equalsIgnoreCase(dryrunSpec)) {
dispenser = new DryRunOpDispenserWrapper((DriverAdapter<Op,Object>)adapter, pop, dispenser);
dryrunCount++;
} else if ("emit".equalsIgnoreCase(dryrunSpec)) {
dispenser = new EmitterOpDispenserWrapper(
(DriverAdapter<Op,Object>)adapter,
pop,
(OpDispenser<? extends CycleOp<?>>) dispenser
);
}
// if (strict) {
@ -502,27 +480,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
}
protected <O extends Op> OpSequence<OpDispenser<? extends O>> createOpSourceFromCommands(
Function<ParsedOp, OpDispenser<? extends O>> opinit,
NBConfiguration cfg,
List<Function<Map<String, Object>, Map<String, Object>>> parsers,
boolean strict
) {
Function<OpTemplate, ParsedOp> f = t -> new ParsedOp(t, cfg, parsers, this);
Function<OpTemplate, OpDispenser<? extends O>> opTemplateOFunction = f.andThen(opinit);
return createOpSequence(opTemplateOFunction, strict, Optional.empty());
}
protected List<ParsedOp> loadParsedOps(NBConfiguration cfg, Optional<DriverAdapter> defaultAdapter) {
List<ParsedOp> parsedOps = loadOpTemplates(defaultAdapter).stream().map(
ot -> new ParsedOp(ot, cfg, List.of(), this)
).toList();
return parsedOps;
}
protected List<OpTemplate> loadOpTemplates(Optional<DriverAdapter> defaultDriverAdapter) {
protected List<OpTemplate> loadOpTemplates(DriverAdapter<?, ?> defaultDriverAdapter) {
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
@ -531,20 +489,20 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
List<OpTemplate> unfilteredOps = opsDocList.getOps(false);
List<OpTemplate> filteredOps = opsDocList.getOps(tagfilter, true);
if (0 == filteredOps.size()) {
if (filteredOps.isEmpty()) {
// There were no ops, and it *wasn't* because they were all filtered out.
// In this case, let's try to synthesize the ops as long as at least a default driver was provided
// But if there were no ops, and there was no default driver provided, we can't continue
// There were no ops, and it was because they were all filtered out
if (0 < unfilteredOps.size()) {
if (!unfilteredOps.isEmpty()) {
throw new BasicError("There were no active op templates with tag filter '"
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
}
if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
if (defaultDriverAdapter instanceof SyntheticOpTemplateProvider sotp) {
filteredOps = sotp.getSyntheticOpTemplates(opsDocList, this.activityDef.getParams());
Objects.requireNonNull(filteredOps);
if (0 == filteredOps.size()) {
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + '\'' +
if (filteredOps.isEmpty()) {
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.getAdapterName() + '\'' +
" but no ops were created. You must provide either a workload or an op parameter. Activities require op templates.");
}
} else {
@ -554,29 +512,28 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
2) op='inline template'
3) driver=stdout (or any other drive that can synthesize ops)""");
}
if (0 == filteredOps.size()) {
throw new BasicError("There were no active op templates with tag filter '" + tagfilter + '\'');
}
}
if (0 == filteredOps.size()) {
throw new OpConfigError("No op templates found. You must provide either workload=... or op=..., or use " +
"a default driver (driver=___). This includes " +
ServiceLoader.load(DriverAdapter.class).stream()
.filter(p -> {
AnnotatedType[] annotatedInterfaces = p.type().getAnnotatedInterfaces();
for (AnnotatedType ai : annotatedInterfaces) {
if (ai.getType().equals(SyntheticOpTemplateProvider.class)) {
return true;
}
}
return false;
})
.map(d -> d.get().getAdapterName())
.collect(Collectors.joining(",")));
}
}
// if (filteredOps.isEmpty()) {
// throw new BasicError("There were no active op templates with tag filter '" + tagfilter + '\'');
// }
// if (filteredOps.isEmpty()) {
// throw new OpConfigError("No op templates found. You must provide either workload=... or op=..., or use " +
// "a default driver (driver=___). This includes " +
// ServiceLoader.load(DriverAdapter.class).stream()
// .filter(p -> {
// AnnotatedType[] annotatedInterfaces = p.type().getAnnotatedInterfaces();
// for (AnnotatedType ai : annotatedInterfaces) {
// if (ai.getType().equals(SyntheticOpTemplateProvider.class)) {
// return true;
// }
// }
// return false;
// })
// .map(d -> d.get().getAdapterName())
// .collect(Collectors.joining(",")));
// }
//
return filteredOps;
}
@ -602,18 +559,17 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
* @param opinit
* A function to map an OpTemplate to the executable operation form required by
* the native driver for this activity.
* @param defaultAdapter
* @param defaultAdapter The adapter which will be used for any op templates with no explicit adapter
* @return The sequence of operations as determined by filtering and ratios
*/
@Deprecated(forRemoval = true)
protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit, boolean strict, Optional<DriverAdapter> defaultAdapter) {
protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit, boolean strict, DriverAdapter<?, ?> defaultAdapter) {
var stmts = loadOpTemplates(defaultAdapter);
List<Long> ratios = new ArrayList<>(stmts.size());
for (int i = 0; i < stmts.size(); i++) {
OpTemplate opTemplate = stmts.get(i);
for (OpTemplate opTemplate : stmts) {
long ratio = opTemplate.removeParamOrDefault("ratio", 1);
ratios.add(ratio);
}
@ -644,22 +600,38 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
protected OpsDocList loadStmtsDocList() {
try {
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
String op = stmt.get();
workloadSource = "commandline:" + stmt.get();
if (op.startsWith("{") || op.startsWith("[")) {
return OpsLoader.loadString(stmt.get(), OpTemplateFormat.json, activityDef.getParams(), null);
} else {
return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null);
}
}
if (op_yaml_loc.isPresent()) {
workloadSource = "yaml:" + op_yaml_loc.get();
return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");
String op = activityDef.getParams().getOptionalString("op").orElse(null);
String stmt = activityDef.getParams().getOptionalString("stmt", "statement").orElse(null);
String workload = activityDef.getParams().getOptionalString("workload").orElse(null);
if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) {
throw new OpConfigError("Only op, statement, or workload may be provided, not more than one.");
}
if (workload != null && OpsLoader.isJson(workload)) {
workloadSource = "commandline: (workload/json):" + workload;
return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null);
} else if (workload != null && OpsLoader.isYaml(workload)) {
workloadSource = "commandline: (workload/yaml):" + workload;
return OpsLoader.loadString(workload, OpTemplateFormat.yaml, activityDef.getParams(), null);
} else if (workload != null) {
return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
}
if (stmt != null) {
workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
}
if (op != null && OpsLoader.isJson(op)) {
workloadSource = "commandline: (op/json): '" + op + "'";
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
}
else if (op != null) {
workloadSource = "commandline: (op/inline): '" + op + "'";
return OpsLoader.loadString(op, OpTemplateFormat.inline, activityDef.getParams(), null);
}
return OpsDocList.none();
} catch (Exception e) {

4
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/input/AtomicInput.java
View File

@ -18,8 +18,8 @@ package io.nosqlbench.engine.api.activityimpl.input;
import com.codahale.metrics.Gauge;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.CyclesSpec;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.cyclelog.buffers.results.CycleSegment;
import io.nosqlbench.engine.api.activityapi.input.Input;

45
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -16,7 +16,6 @@
package io.nosqlbench.engine.api.activityimpl.uniform;
import com.codahale.metrics.Gauge;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
@ -28,7 +27,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTem
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.nb.api.lifecycle.Shutdownable;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.config.standard.*;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.errors.BasicError;
@ -59,21 +58,15 @@ import java.util.concurrent.ConcurrentHashMap;
*/
public class StandardActivity<R extends Op, S> extends SimpleActivity implements SyntheticOpTemplateProvider, ActivityDefObserver {
private static final Logger logger = LogManager.getLogger("ACTIVITY");
private final OpSequence<OpDispenser<? extends Op>> sequence;
private final NBConfigModel yamlmodel;
private final ConcurrentHashMap<String, DriverAdapter> adapters = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, OpMapper<Op>> mappers = new ConcurrentHashMap<>();
private final Gauge<Double> pendingOpsGauge;
private final Gauge<Double> activeOpsGauge;
private final Gauge<Double> completeOpsGauge;
private final ConcurrentHashMap<String, DriverAdapter<?,?>> adapters = new ConcurrentHashMap<>();
public StandardActivity(NBComponent parent, ActivityDef activityDef) {
super(parent,activityDef);
OpsDocList workload;
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
NBConfigModel yamlmodel;
if (yaml_loc.isPresent()) {
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities");
@ -83,7 +76,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
Optional<String> defaultDriverName = activityDef.getParams().getOptionalString("driver");
Optional<DriverAdapter> defaultAdapter = activityDef.getParams().getOptionalString("driver")
Optional<DriverAdapter<?,?>> defaultAdapter = activityDef.getParams().getOptionalString("driver")
.flatMap(name -> ServiceSelector.of(name,ServiceLoader.load(DriverAdapterLoader.class)).get())
.map(l -> l.load(this,NBLabels.forKV()));
@ -92,14 +85,15 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
// HERE, op templates are loaded before drivers are loaded
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter);
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter.orElse(null));
List<ParsedOp> pops = new ArrayList<>();
List<DriverAdapter> adapterlist = new ArrayList<>();
List<DriverAdapter<?,?>> adapterlist = new ArrayList<>();
NBConfigModel supersetConfig = ConfigModel.of(StandardActivity.class).add(yamlmodel);
Optional<String> defaultDriverOption = activityDef.getParams().getOptionalString("driver");
ConcurrentHashMap<String, OpMapper<? extends Op>> mappers = new ConcurrentHashMap<>();
for (OpTemplate ot : opTemplates) {
// ParsedOp incompleteOpDef = new ParsedOp(ot, NBConfiguration.empty(), List.of(), this);
String driverName = ot.getOptionalStringParam("driver", String.class)
@ -116,7 +110,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
// HERE
if (!adapters.containsKey(driverName)) {
DriverAdapter adapter = Optional.of(driverName)
DriverAdapter<?,?> adapter = Optional.of(driverName)
.flatMap(name -> ServiceSelector.of(name,ServiceLoader.load(DriverAdapterLoader.class)).get())
.map(l -> l.load(this,NBLabels.forKV())).orElseThrow();
@ -137,7 +131,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
supersetConfig.assertValidConfig(activityDef.getParams().getStringStringMap());
DriverAdapter adapter = adapters.get(driverName);
DriverAdapter<?,?> adapter = adapters.get(driverName);
adapterlist.add(adapter);
ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()), this);
Optional<String> discard = pop.takeOptionalStaticValue("driver", String.class);
@ -152,8 +146,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
try {
boolean strict = activityDef.getParams().getOptionalBoolean("strict").orElse(false);
sequence = createOpSourceFromParsedOps(adapters, mappers, adapterlist, pops);
sequence = createOpSourceFromParsedOps(adapterlist, pops);
} catch (Exception e) {
if (e instanceof OpConfigError) {
throw e;
@ -161,12 +154,12 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
throw new OpConfigError("Error mapping workload template to operations: " + e.getMessage(), null, e);
}
this.pendingOpsGauge = create().gauge(
"ops_pending",() -> this.getProgressMeter().getSummary().pending());
this.activeOpsGauge = create().gauge(
"ops_active",() -> this.getProgressMeter().getSummary().current());
this.completeOpsGauge = create().gauge(
"ops_complete",() -> this.getProgressMeter().getSummary().complete());
create().gauge(
"ops_pending", () -> this.getProgressMeter().getSummary().pending());
create().gauge(
"ops_active", () -> this.getProgressMeter().getSummary().current());
create().gauge(
"ops_complete", () -> this.getProgressMeter().getSummary().complete());
}
@Override
@ -196,7 +189,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
public synchronized void onActivityDefUpdate(ActivityDef activityDef) {
super.onActivityDefUpdate(activityDef);
for (DriverAdapter adapter : adapters.values()) {
for (DriverAdapter<?,?> adapter : adapters.values()) {
if (adapter instanceof NBReconfigurable configurable) {
NBConfigModel cfgModel = configurable.getReconfigModel();
NBConfiguration cfg = cfgModel.matchConfig(activityDef.getParams());
@ -227,7 +220,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
@Override
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
List<OpTemplate> opTemplates = new ArrayList<>();
for (DriverAdapter adapter : adapters.values()) {
for (DriverAdapter<?,?> adapter : adapters.values()) {
if (adapter instanceof SyntheticOpTemplateProvider sotp) {
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(opsDocList, cfg);
opTemplates.addAll(newTemplates);
@ -243,7 +236,7 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
*/
@Override
public void shutdownActivity() {
for (Map.Entry<String, DriverAdapter> entry : adapters.entrySet()) {
for (Map.Entry<String, DriverAdapter<?,?>> entry : adapters.entrySet()) {
String adapterName = entry.getKey();
DriverAdapter<?, ?> adapter = entry.getValue();
adapter.getSpaceCache().getElements().forEach((spaceName, space) -> {

2
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;

2
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/actions/StandardAction.java
View File

@ -103,7 +103,7 @@ public class StandardAction<A extends StandardActivity<R, ?>, R extends Op> impl
if (op instanceof RunnableOp) {
((RunnableOp) op).run();
} else if (op instanceof CycleOp<?>) {
result = ((CycleOp) op).apply(cycle);
result = ((CycleOp<?>) op).apply(cycle);
} else if (op instanceof ChainingOp) {
result = ((ChainingOp) op).apply(result);
} else {

2
engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionCountMetrics.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Counter;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.util.ArrayList;
import java.util.List;

2
engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionExpectedResultVerificationMetrics.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Counter;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
/**

2
engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionHistoMetrics.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Histogram;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.util.ArrayList;
import java.util.List;

2
engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionMeterMetrics.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Meter;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.util.ArrayList;
import java.util.List;

2
engine-core/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionTimerMetrics.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.api.metrics;
import com.codahale.metrics.Timer;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.util.ArrayList;
import java.util.List;

2
engine-core/src/main/java/io/nosqlbench/engine/cmdstream/NBJavaCommandLoader.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.cmdstream;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandInfo;
import io.nosqlbench.nb.annotations.ServiceSelector;

4
engine-core/src/main/java/io/nosqlbench/engine/core/clientload/ClientSystemMetricChecker.java
View File

@ -18,8 +18,8 @@ package io.nosqlbench.engine.core.clientload;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.concurrent.Executors;

8
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ExecutionMetricsResult.java
View File

@ -23,10 +23,10 @@ import com.codahale.metrics.MetricFilter;
import io.nosqlbench.nb.api.engine.metrics.instruments.*;
import io.nosqlbench.nb.api.engine.metrics.reporters.ConsoleReporter;
import io.nosqlbench.nb.api.engine.metrics.reporters.Log4JMetricsReporter;
import io.nosqlbench.nb.api.components.NBCreators;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.NBComponentTraversal;
import io.nosqlbench.nb.api.components.NBFinders;
import io.nosqlbench.nb.api.components.core.NBCreators;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponentTraversal;
import io.nosqlbench.nb.api.components.core.NBFinders;
import io.nosqlbench.engine.core.metrics.NBMetricsSummary;
import java.io.ByteArrayOutputStream;

7
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivitiesProgressIndicator.java
View File

@ -80,6 +80,10 @@ public class ActivitiesProgressIndicator implements Runnable {
@Override
public void run() {
report();
}
private void report() {
Collection<ProgressMeterDisplay> progressMeterDisplays = sc.getProgressMeters();
for (ProgressMeterDisplay meter : progressMeterDisplays) {
@ -117,4 +121,7 @@ public class ActivitiesProgressIndicator implements Runnable {
return "ProgressIndicator/" + this.indicatorSpec;
}
public void finish() {
report();
}
}

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java
View File

@ -19,7 +19,7 @@ import com.codahale.metrics.Gauge;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBComponentExecutionScope;
import io.nosqlbench.nb.api.components.core.NBComponentExecutionScope;
import io.nosqlbench.engine.api.activityapi.core.*;
import io.nosqlbench.engine.api.activityimpl.MotorState;
import io.nosqlbench.nb.api.annotations.Annotation;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityLoader.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.core.lifecycle.activity;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.api.activityapi.core.Activity;
import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType;
import org.apache.logging.log4j.LogManager;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityTypeLoader.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.core.lifecycle.activity;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.nbio.Content;
import io.nosqlbench.nb.api.nbio.NBIO;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;

6
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/container/ContainerActivitiesController.java
View File

@ -17,9 +17,9 @@ package io.nosqlbench.engine.core.lifecycle.scenario.container;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.activityimpl.ParameterMap;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.NBComponentErrorHandler;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponentErrorHandler;
import io.nosqlbench.engine.api.activityapi.core.Activity;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
import io.nosqlbench.engine.core.lifecycle.ExecutionResult;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/container/ContainerBuilderFacets.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.core.lifecycle.scenario.container;
*/
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.io.PrintWriter;
import java.io.Reader;

10
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/container/NBBufferedContainer.java
View File

@ -24,8 +24,8 @@ import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand
import io.nosqlbench.nb.api.annotations.Annotation;
import io.nosqlbench.nb.api.annotations.Layer;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -95,7 +95,8 @@ public class NBBufferedContainer extends NBBaseComponent implements NBContainer
.build()
);
this.activitiesProgressIndicator = new ActivitiesProgressIndicator(this.controller, "console:10s");
String progress = getComponentProp("progress").orElse("console:10s");
this.activitiesProgressIndicator = new ActivitiesProgressIndicator(this.controller, progress);
}
@ -175,6 +176,7 @@ public class NBBufferedContainer extends NBBaseComponent implements NBContainer
logger.debug("no object was provided to set the container result");
}
activitiesProgressIndicator.finish();
return safeCmdResult;
}
@ -195,6 +197,6 @@ public class NBBufferedContainer extends NBBaseComponent implements NBContainer
final var retiringScenarioShutdownHook = this.containerShutdownHook;
this.containerShutdownHook = null;
retiringScenarioShutdownHook.run();
this.logger.debug("removing container shutdown hook");
logger.debug("removing container shutdown hook");
}
}

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/container/NBContainer.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.core.lifecycle.scenario.container;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBCommandResult;
import io.nosqlbench.engine.core.lifecycle.scenario.execution.NBInvokableCommand;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.io.PrintWriter;
import java.io.Reader;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/container/NBScenarioContainerBuilder.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.core.lifecycle.scenario.container;
*/
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import java.io.InputStreamReader;
import java.io.PrintWriter;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/execution/NBCommandInfo.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.core.lifecycle.scenario.execution;
import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/execution/NBInvokableCommand.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.engine.core.lifecycle.scenario.execution;
import io.nosqlbench.engine.core.lifecycle.scenario.container.NBBufferedContainer;
import io.nosqlbench.engine.core.lifecycle.scenario.container.NBCommandParams;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.labels.NBLabels;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/execution/ScenarioSummary.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.engine.core.lifecycle.scenario.execution;
import io.nosqlbench.nb.api.components.NBComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

6
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBCommandAssembly.java
View File

@ -46,11 +46,11 @@ public class NBCommandAssembly {
for (Cmd cmd : cmds) {
if (cmd.getArgs().containsKey("container")) {
String ctx = cmd.getArgs().remove("container").getValue();
String specificContainer = cmd.getArgs().remove("container").getValue();
String step = cmd.getArgs().containsKey("step") ? cmd.getArgs().remove("step").getValue() : "no-step";
tagged.add(cmd.forContainer(ctx, step));
tagged.add(cmd.forContainer(specificContainer, step));
} else if (cmd.getCmdType() == CmdType.container) {
containerName = cmd.getArgValue("container");
containerName = cmd.getArgValue("name");
if (containerName.equals(Cmd.DEFAULT_TARGET_CONTEXT)) {
logger.warn("You are explicitly setting the scenario name to " + Cmd.DEFAULT_TARGET_CONTEXT + "'. This is likely an error. " +
"This is the default scenario name, and if you are using different scenario names you should pick something that is different and specific.");

17
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/session/NBSession.java
View File

@ -16,12 +16,13 @@
package io.nosqlbench.engine.core.lifecycle.session;
import io.nosqlbench.nb.api.components.core.NBComponentProps;
import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBFunctionGauge;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.nb.api.labels.NBLabeledElement;
import io.nosqlbench.nb.api.labels.NBLabels;
import io.nosqlbench.nb.api.components.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.decorators.NBTokenWords;
import io.nosqlbench.engine.cmdstream.Cmd;
import io.nosqlbench.engine.core.clientload.*;
@ -55,13 +56,6 @@ public class NBSession extends NBBaseComponent implements Function<List<Cmd>, Ex
ERROR
}
private NBBufferedContainer getContext(String name) {
return containers.computeIfAbsent(
name,
n -> NBContainer.builder().name(n).build(this)
);
}
public NBSession(
NBLabeledElement labelContext,
String sessionName
@ -205,5 +199,12 @@ public class NBSession extends NBBaseComponent implements Function<List<Cmd>, Ex
clientMetricChecker.addRatioMetricToCheck(cpuUserGauge, cpuTotalGauge, 50.0, true);
}
private NBBufferedContainer getContext(String name) {
return containers.computeIfAbsent(
name,
n -> NBContainer.builder().name(n).build(this)
);
}
}

Some files were not shown because too many files have changed in this diff Show More