mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
add dryrun=emit support
This commit is contained in:
Jonathan Shook
parent
3547c04c41
commit
5ab00a2c11
@ -99,7 +99,7 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
public final List<Row> apply(long cycle) {
|
||||
public final ArrayList<Row> apply(long cycle) {
|
||||
|
||||
Statement<?> statement = getStmt();
|
||||
logger.trace(() -> "apply() invoked, statement obtained, executing async with page size: " + statement.getPageSize() + " thread local rows: ");
|
||||
@ -118,7 +118,7 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
|
||||
});
|
||||
|
||||
try {
|
||||
return rowsStage.toCompletableFuture().get(300, TimeUnit.SECONDS);
|
||||
return new PrintableRowList(rowsStage.toCompletableFuture().get(300, TimeUnit.SECONDS));
|
||||
} catch (ExecutionException exe) {
|
||||
Throwable ee = exe.getCause();
|
||||
if (ee instanceof RuntimeException re) {
|
||||
@ -142,6 +142,21 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
|
||||
// processors.flush();
|
||||
}
|
||||
|
||||
private static class PrintableRowList extends ArrayList<Row> {
|
||||
public PrintableRowList(List<Row> values) {
|
||||
super(values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Row row : this) {
|
||||
sb.append(row.getFormattedContents());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
// private BiFunction<AsyncResultSet,Throwable> handler
|
||||
@Override
|
||||
public Op getNextOp() {
|
||||
|
||||
@ -176,7 +176,7 @@ public abstract class BaseDriverAdapter<R extends Op, S> extends NBBaseComponent
|
||||
.add(Param.optional("instrument", Boolean.class))
|
||||
.add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
|
||||
.add(Param.optional("driver", String.class))
|
||||
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|none)"))
|
||||
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
|
||||
.add(Param.optional("maxtries", Integer.class))
|
||||
.asReadOnly();
|
||||
}
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.adapters.api.activityimpl.uniform;
|
||||
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
|
||||
|
||||
public class EmitterOp implements CycleOp<Object> {
|
||||
|
||||
private final CycleOp<?> cycleOp;
|
||||
public EmitterOp(CycleOp<?> cycleOp) {
|
||||
this.cycleOp = cycleOp;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object apply(long value) {
|
||||
Object result = cycleOp.apply(value);
|
||||
System.out.println("result from cycle " + value + ":\n"+result);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.adapters.api.activityimpl.uniform;
|
||||
|
||||
import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
|
||||
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
|
||||
public class EmitterOpDispenserWrapper extends BaseOpDispenser<Op, Object> {
|
||||
|
||||
private final OpDispenser<? extends CycleOp<?>> realDispenser;
|
||||
|
||||
public EmitterOpDispenserWrapper(DriverAdapter<Op,Object> adapter, ParsedOp pop, OpDispenser<? extends CycleOp<?>> realDispenser) {
|
||||
super(adapter, pop);
|
||||
this.realDispenser = realDispenser;
|
||||
}
|
||||
@Override
|
||||
public EmitterOp apply(long cycle) {
|
||||
CycleOp<?> cycleOp = realDispenser.apply(cycle);
|
||||
return new EmitterOp(cycleOp);
|
||||
}
|
||||
}
|
||||
@ -16,6 +16,8 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityimpl;
|
||||
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.EmitterOpDispenserWrapper;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
|
||||
import io.nosqlbench.nb.api.components.core.NBComponent;
|
||||
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
|
||||
import io.nosqlbench.nb.api.components.events.ParamChange;
|
||||
@ -56,12 +58,10 @@ import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.lang.reflect.AnnotatedType;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* A default implementation of an Activity, suitable for building upon.
|
||||
@ -71,7 +71,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
|
||||
protected ActivityDef activityDef;
|
||||
private final List<AutoCloseable> closeables = new ArrayList<>();
|
||||
private MotorDispenser motorDispenser;
|
||||
private MotorDispenser<?> motorDispenser;
|
||||
private InputDispenser inputDispenser;
|
||||
private ActionDispenser actionDispenser;
|
||||
private OutputDispenser markerDispenser;
|
||||
@ -121,7 +121,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
if (null == this.errorHandler) {
|
||||
errorHandler = new NBErrorHandler(
|
||||
() -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
|
||||
() -> getExceptionMetrics());
|
||||
this::getExceptionMetrics);
|
||||
}
|
||||
return errorHandler;
|
||||
}
|
||||
@ -316,6 +316,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
public void createOrUpdateStrideLimiter(SimRateSpec spec) {
|
||||
strideLimiter = RateLimiters.createOrUpdate(this, strideLimiter, spec);
|
||||
}
|
||||
|
||||
public void createOrUpdateCycleLimiter(SimRateSpec spec) {
|
||||
cycleLimiter = RateLimiters.createOrUpdate(this, cycleLimiter, spec);
|
||||
}
|
||||
@ -409,24 +410,23 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
" more information.");
|
||||
}
|
||||
|
||||
if (0 < this.activityDef.getCycleCount() && 0 == seq.getOps().size()) {
|
||||
if (0 < this.activityDef.getCycleCount() && seq.getOps().isEmpty()) {
|
||||
throw new BasicError("You have configured a zero-length sequence and non-zero cycles. Tt is not possible to continue with this activity.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected <O extends Op> OpSequence<OpDispenser<? extends O>> createOpSourceFromParsedOps(
|
||||
Map<String, DriverAdapter> adapterCache,
|
||||
Map<String, OpMapper<Op>> mapperCache,
|
||||
List<DriverAdapter> adapters,
|
||||
// Map<String, DriverAdapter<?,?>> adapterCache,
|
||||
// Map<String, OpMapper<? extends Op>> mapperCache,
|
||||
List<DriverAdapter<?,?>> adapters,
|
||||
List<ParsedOp> pops
|
||||
) {
|
||||
try {
|
||||
|
||||
List<Long> ratios = new ArrayList<>(pops.size());
|
||||
|
||||
for (int i = 0; i < pops.size(); i++) {
|
||||
ParsedOp pop = pops.get(i);
|
||||
for (ParsedOp pop : pops) {
|
||||
long ratio = pop.takeStaticConfigOr("ratio", 1);
|
||||
ratios.add(ratio);
|
||||
}
|
||||
@ -445,16 +445,21 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
logger.info(() -> "skipped mapping op '" + pop.getName() + '\'');
|
||||
continue;
|
||||
}
|
||||
String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
|
||||
boolean dryrun = "op".equalsIgnoreCase(dryrunSpec);
|
||||
|
||||
DriverAdapter adapter = adapters.get(i);
|
||||
OpMapper opMapper = adapter.getOpMapper();
|
||||
DriverAdapter<?,?> adapter = adapters.get(i);
|
||||
OpMapper<? extends Op> opMapper = adapter.getOpMapper();
|
||||
OpDispenser<? extends Op> dispenser = opMapper.apply(pop);
|
||||
|
||||
if (dryrun) {
|
||||
dispenser = new DryRunOpDispenserWrapper(adapter, pop, dispenser);
|
||||
String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
|
||||
if ("op".equalsIgnoreCase(dryrunSpec)) {
|
||||
dispenser = new DryRunOpDispenserWrapper((DriverAdapter<Op,Object>)adapter, pop, dispenser);
|
||||
dryrunCount++;
|
||||
} else if ("emit".equalsIgnoreCase(dryrunSpec)) {
|
||||
dispenser = new EmitterOpDispenserWrapper(
|
||||
(DriverAdapter<Op,Object>)adapter,
|
||||
pop,
|
||||
(OpDispenser<? extends CycleOp<?>>) dispenser
|
||||
);
|
||||
}
|
||||
|
||||
// if (strict) {
|
||||
@ -475,7 +480,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
|
||||
}
|
||||
|
||||
protected List<OpTemplate> loadOpTemplates(Optional<DriverAdapter<?,?>> defaultDriverAdapter) {
|
||||
protected List<OpTemplate> loadOpTemplates(DriverAdapter<?, ?> defaultDriverAdapter) {
|
||||
|
||||
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
|
||||
|
||||
@ -493,11 +498,11 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
throw new BasicError("There were no active op templates with tag filter '"
|
||||
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
|
||||
}
|
||||
if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
|
||||
if (defaultDriverAdapter instanceof SyntheticOpTemplateProvider sotp) {
|
||||
filteredOps = sotp.getSyntheticOpTemplates(opsDocList, this.activityDef.getParams());
|
||||
Objects.requireNonNull(filteredOps);
|
||||
if (filteredOps.isEmpty()) {
|
||||
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + '\'' +
|
||||
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.getAdapterName() + '\'' +
|
||||
" but no ops were created. You must provide either a workload or an op parameter. Activities require op templates.");
|
||||
}
|
||||
} else {
|
||||
@ -507,28 +512,28 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
2) op='inline template'
|
||||
3) driver=stdout (or any other drive that can synthesize ops)""");
|
||||
}
|
||||
if (filteredOps.isEmpty()) {
|
||||
throw new BasicError("There were no active op templates with tag filter '" + tagfilter + '\'');
|
||||
}
|
||||
}
|
||||
|
||||
if (filteredOps.isEmpty()) {
|
||||
throw new OpConfigError("No op templates found. You must provide either workload=... or op=..., or use " +
|
||||
"a default driver (driver=___). This includes " +
|
||||
ServiceLoader.load(DriverAdapter.class).stream()
|
||||
.filter(p -> {
|
||||
AnnotatedType[] annotatedInterfaces = p.type().getAnnotatedInterfaces();
|
||||
for (AnnotatedType ai : annotatedInterfaces) {
|
||||
if (ai.getType().equals(SyntheticOpTemplateProvider.class)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
})
|
||||
.map(d -> d.get().getAdapterName())
|
||||
.collect(Collectors.joining(",")));
|
||||
}
|
||||
// if (filteredOps.isEmpty()) {
|
||||
// throw new BasicError("There were no active op templates with tag filter '" + tagfilter + '\'');
|
||||
// }
|
||||
|
||||
// if (filteredOps.isEmpty()) {
|
||||
// throw new OpConfigError("No op templates found. You must provide either workload=... or op=..., or use " +
|
||||
// "a default driver (driver=___). This includes " +
|
||||
// ServiceLoader.load(DriverAdapter.class).stream()
|
||||
// .filter(p -> {
|
||||
// AnnotatedType[] annotatedInterfaces = p.type().getAnnotatedInterfaces();
|
||||
// for (AnnotatedType ai : annotatedInterfaces) {
|
||||
// if (ai.getType().equals(SyntheticOpTemplateProvider.class)) {
|
||||
// return true;
|
||||
// }
|
||||
// }
|
||||
// return false;
|
||||
// })
|
||||
// .map(d -> d.get().getAdapterName())
|
||||
// .collect(Collectors.joining(",")));
|
||||
// }
|
||||
//
|
||||
return filteredOps;
|
||||
}
|
||||
|
||||
@ -554,18 +559,17 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
* @param opinit
|
||||
* A function to map an OpTemplate to the executable operation form required by
|
||||
* the native driver for this activity.
|
||||
* @param defaultAdapter
|
||||
* @param defaultAdapter The adapter which will be used for any op templates with no explicit adapter
|
||||
* @return The sequence of operations as determined by filtering and ratios
|
||||
*/
|
||||
@Deprecated(forRemoval = true)
|
||||
protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit, boolean strict, Optional<DriverAdapter<?,?>> defaultAdapter) {
|
||||
protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit, boolean strict, DriverAdapter<?, ?> defaultAdapter) {
|
||||
|
||||
var stmts = loadOpTemplates(defaultAdapter);
|
||||
|
||||
List<Long> ratios = new ArrayList<>(stmts.size());
|
||||
|
||||
for (int i = 0; i < stmts.size(); i++) {
|
||||
OpTemplate opTemplate = stmts.get(i);
|
||||
for (OpTemplate opTemplate : stmts) {
|
||||
long ratio = opTemplate.removeParamOrDefault("ratio", 1);
|
||||
ratios.add(ratio);
|
||||
}
|
||||
@ -599,29 +603,37 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
|
||||
String op = activityDef.getParams().getOptionalString("op").orElse(null);
|
||||
String stmt = activityDef.getParams().getOptionalString("stmt", "statement").orElse(null);
|
||||
String workload = activityDef.getParams().getOptionalString("workload").orElse(null);
|
||||
|
||||
if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) {
|
||||
throw new OpConfigError("Only op, statement, or workload may be provided, not more than one.");
|
||||
}
|
||||
|
||||
if (op!=null && op.matches("^\\{[^}]+:[^}]+}$(?s)(?m)")) {
|
||||
workloadSource = "commandline: (op/json): '" + op + "'";
|
||||
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
|
||||
} else if (op!=null && op.matches("^\\[[^]]+]$")) {
|
||||
workloadSource = "commandline: (op/json): '" + op + "'";
|
||||
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
|
||||
} else if (op!=null) {
|
||||
workloadSource = "commandline: (op/inline): '" + op + "'";
|
||||
return OpsLoader.loadString(op, OpTemplateFormat.inline, activityDef.getParams(), null);
|
||||
} else if (stmt!=null) {
|
||||
|
||||
if (workload != null && OpsLoader.isJson(workload)) {
|
||||
workloadSource = "commandline: (workload/json):" + workload;
|
||||
return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null);
|
||||
} else if (workload != null && OpsLoader.isYaml(workload)) {
|
||||
workloadSource = "commandline: (workload/yaml):" + workload;
|
||||
return OpsLoader.loadString(workload, OpTemplateFormat.yaml, activityDef.getParams(), null);
|
||||
} else if (workload != null) {
|
||||
return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
|
||||
}
|
||||
|
||||
if (stmt != null) {
|
||||
workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
|
||||
return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
|
||||
} else if (workload!=null) {
|
||||
workloadSource = "yaml:" + workload;
|
||||
return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
|
||||
} else {
|
||||
return OpsDocList.none();
|
||||
}
|
||||
|
||||
if (op != null && OpsLoader.isJson(op)) {
|
||||
workloadSource = "commandline: (op/json): '" + op + "'";
|
||||
return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
|
||||
}
|
||||
else if (op != null) {
|
||||
workloadSource = "commandline: (op/inline): '" + op + "'";
|
||||
return OpsLoader.loadString(op, OpTemplateFormat.inline, activityDef.getParams(), null);
|
||||
}
|
||||
return OpsDocList.none();
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user