IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

jshook/nosqlbench-2068-opspaces (#2069)

Browse Source 
* force mappers to use op-contextual space bindings

* move space de-init to owner (adapter component)

* make specific adapters use op-contextual spaces
This commit is contained in:
Jonathan Shook
2024-11-05 16:59:28 -06:00
committed by GitHub
parent ea7fceae49
commit 722e8b53d8
40 changed files with 237 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/BaseOpDispenser.java
View File

@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.LongFunction;
/**
* See {@link OpDispenser} for details on how to use this type.
@@ -65,6 +66,7 @@ public abstract class BaseOpDispenser<OP extends CycleOp<?>,SPACE extends Space>
private Timer errorTimer;
private final String[] timerStarts;
private final String[] timerStops;
protected LongFunction<? extends SPACE> spaceF;
/**
* package imports used with "verifiers" or "expected-result" are accumulated here
@@ -83,6 +85,7 @@ public abstract class BaseOpDispenser<OP extends CycleOp<?>,SPACE extends Space>
super(adapter);
opName = op.getName();
this.adapter = adapter;
this.spaceF = adapter.getSpaceFunc(op);
labels = op.getLabels();
this.timerStarts = op.takeOptionalStaticValue(START_TIMERS, String.class)

2
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/OpMapper.java
View File

@@ -82,7 +82,7 @@ import java.util.function.LongFunction;
* generally something that implements {@link Runnable}.
*/
public interface OpMapper<OPTYPE extends CycleOp<?>, SPACETYPE extends Space>
extends BiFunction<ParsedOp, LongFunction<SPACETYPE>, OpDispenser<OPTYPE>> {
extends BiFunction<ParsedOp, LongFunction<SPACETYPE>, OpDispenser<? extends OPTYPE>> {
/**
* Interrogate the parsed command, and provide a new

89
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java
View File

@@ -35,12 +35,15 @@ import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
import java.util.stream.Collectors;
public abstract class BaseDriverAdapter<R extends CycleOp<?>, S extends Space> extends NBBaseComponent implements DriverAdapter<R, S>, NBConfigurable, NBReconfigurable {
public abstract class BaseDriverAdapter<RESULT
extends CycleOp<?>, SPACE extends Space> extends NBBaseComponent
implements DriverAdapter<RESULT, SPACE>, NBConfigurable, NBReconfigurable {
private final static Logger logger = LogManager.getLogger("ADAPTER");
private ConcurrentSpaceCache<S> spaceCache;
private ConcurrentSpaceCache<SPACE> spaceCache;
private NBConfiguration cfg;
private LongFunction<S> spaceF;
private LongFunction<SPACE> spaceF;
public BaseDriverAdapter(NBComponent parentComponent, NBLabels labels) {
super(parentComponent, labels);
@@ -57,9 +60,9 @@ public abstract class BaseDriverAdapter<R extends CycleOp<?>, S extends Space> e
public final Function<Map<String, Object>, Map<String, Object>> getPreprocessor() {
List<Function<Map<String, Object>, Map<String, Object>>> mappers = new ArrayList<>();
List<Function<Map<String, Object>, Map<String, Object>>> stmtRemappers =
getOpStmtRemappers().stream()
.map(m -> new FieldDestructuringMapper("stmt", m))
.collect(Collectors.toList());
getOpStmtRemappers().stream()
.map(m -> new FieldDestructuringMapper("stmt", m))
.collect(Collectors.toList());
mappers.addAll(stmtRemappers);
mappers.addAll(getOpFieldRemappers());
@@ -129,10 +132,9 @@ public abstract class BaseDriverAdapter<R extends CycleOp<?>, S extends Space> e
return List.of();
}
@Override
public final synchronized ConcurrentSpaceCache<S> getSpaceCache() {
private final synchronized ConcurrentSpaceCache<SPACE> getSpaceCache() {
if (spaceCache == null) {
spaceCache = new ConcurrentSpaceCache<S>(this,getSpaceInitializer(getConfiguration()));
spaceCache = new ConcurrentSpaceCache<SPACE>(this, getSpaceInitializer(getConfiguration()));
}
return spaceCache;
}
@@ -161,40 +163,40 @@ public abstract class BaseDriverAdapter<R extends CycleOp<?>, S extends Space> e
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(BaseDriverAdapter.class)
.add(Param.optional("alias"))
.add(Param.optional("labels",String.class,"Labels which will apply to metrics and annotations for this activity only"))
.add(Param.defaultTo("strict", true, "strict op field mode, which requires that provided op fields are recognized and used"))
.add(Param.optional(List.of("op", "stmt", "statement"), String.class, "op template in statement form"))
.add(Param.optional("tags", String.class, "tags to be used to filter operations"))
.add(Param.defaultTo("errors", "stop", "error handler configuration"))
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("stride").setRegex("\\d+"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional("cycles").setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\.\\d+[KMBGTPE]?").setDescription("cycle interval to use"))
.add(Param.optional("recycles").setDescription("allow cycles to be re-used this many times"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.add(Param.optional("seq", String.class, "sequencing algorithm"))
.add(Param.optional("instrument", Boolean.class))
.add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
.add(Param.optional("driver", String.class))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
.add(Param.optional("maxtries", Integer.class))
.asReadOnly();
.add(Param.optional("alias"))
.add(Param.optional("labels", String.class, "Labels which will apply to metrics and annotations for this activity only"))
.add(Param.defaultTo("strict", true, "strict op field mode, which requires that provided op fields are recognized and used"))
.add(Param.optional(List.of("op", "stmt", "statement"), String.class, "op template in statement form"))
.add(Param.optional("tags", String.class, "tags to be used to filter operations"))
.add(Param.defaultTo("errors", "stop", "error handler configuration"))
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("stride").setRegex("\\d+"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional("cycles").setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\.\\d+[KMBGTPE]?").setDescription("cycle interval to use"))
.add(Param.optional("recycles").setDescription("allow cycles to be re-used this many times"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.add(Param.optional("seq", String.class, "sequencing algorithm"))
.add(Param.optional("instrument", Boolean.class))
.add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
.add(Param.optional("driver", String.class))
.add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
.add(Param.optional("maxtries", Integer.class))
.asReadOnly();
}
@Override
public NBConfigModel getReconfigModel() {
return ConfigModel.of(BaseDriverAdapter.class)
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.asReadOnly();
.add(Param.optional("threads").setRegex("\\d+|\\d+x|auto").setDescription("number of concurrent operations, controlled by threadpool"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.asReadOnly();
}
@Override
public LongFunction<S> getSpaceFunc(ParsedOp pop) {
public LongFunction<SPACE> getSpaceFunc(ParsedOp pop) {
Optional<LongFunction<Object>> spaceFuncTest = pop.getAsOptionalFunction("space",Object.class);
Optional<LongFunction<Object>> spaceFuncTest = pop.getAsOptionalFunction("space", Object.class);
LongToIntFunction cycleToSpaceF;
if (spaceFuncTest.isEmpty()) {
cycleToSpaceF = (long l) -> 0;
@@ -203,7 +205,7 @@ public abstract class BaseDriverAdapter<R extends CycleOp<?>, S extends Space> e
if (example instanceof Number n) {
logger.trace("mapping space indirectly with Number type");
LongFunction<Number> numberF = pop.getAsRequiredFunction("space", Number.class);
cycleToSpaceF= l -> numberF.apply(l).intValue();
cycleToSpaceF = l -> numberF.apply(l).intValue();
} else {
logger.trace("mapping space indirectly through hash table to index pool");
LongFunction<?> sourceF = pop.getAsRequiredFunction("space", String.class);
@@ -212,7 +214,22 @@ public abstract class BaseDriverAdapter<R extends CycleOp<?>, S extends Space> e
cycleToSpaceF = l -> wrapper.mapKeyToIndex(namerF.apply(l));
}
}
ConcurrentSpaceCache<S> spaceCache1 = getSpaceCache();
ConcurrentSpaceCache<SPACE> spaceCache1 = getSpaceCache();
return l -> spaceCache1.get(cycleToSpaceF.applyAsInt(l));
}
@Override
public void beforeDetach() {
for (SPACE space : this.getSpaceCache()) {
try {
// TODO This should be invariant now, remove conditional?
space.close();
} catch (Exception e) {
throw new RuntimeException("Error while shutting down state space for " +
"adapter=" + this.getAdapterName() + ", space=" + space.getName() + ": " + e, e);
}
}
super.beforeDetach();
}
}

41
nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java
View File

@@ -131,22 +131,7 @@ public interface DriverAdapter<OPTYPE extends CycleOp<?>, SPACETYPE extends Spac
return List.of(f -> f);
}
/**
* The cache of all objects needed within a single instance
* of a DriverAdapter which are not operations. These are generally
* things needed by operations, or things needed during the
* construction of operations.
*
* See {@link ConcurrentIndexCache} for details on when and how to use this function.
*
* <p>During Adapter Initialization, Op Mapping, Op Synthesis, or Op Execution,
* you may need access to the objects in (the or a) space cache. You can build the
* type of context needed and then provide this function to provide new instances
* when needed.</p>
*
* @return A cache of named objects
*/
ConcurrentSpaceCache<SPACETYPE> getSpaceCache();
// ConcurrentSpaceCache<SPACETYPE> getSpaceCache();
/**
* This method allows each driver adapter to create named state which is automatically
@@ -209,5 +194,27 @@ public interface DriverAdapter<OPTYPE extends CycleOp<?>, SPACETYPE extends Spac
return this.getClass().getAnnotation(Service.class).maturity();
}
LongFunction<SPACETYPE> getSpaceFunc(ParsedOp pop);
/**
* <p>The cache of all objects needed within a single instance
* of a DriverAdapter which are not operations. These are generally
* things needed by operations, or things needed during the
* construction of operations.</p>
*
* <p>During Adapter Initialization, Op Mapping, Op Synthesis, or Op Execution,
* you may need access to the objects in (the or a) space cache. You can build the
* type of context needed and then provide this function to provide new instances
* when needed.</p>
*
* <p>The function returned by this method is specialized to the space mapping
* logic in the op template. Specifically, it uses whatever binding is set on a given
* op template for the <em>space</em> op field. If none are provided, then this
* becomes a short-circuit for the default '0'. If a non-numeric binding is provided,
* then an interstitial mapping is added which converts the {@link Object#toString()}
* value to ordinals using a hash map. This is less optimal by far than using
* any binding that produces a {@link Number}.</p>
*
* @return A cache of named objects
*/
public LongFunction<SPACETYPE> getSpaceFunc(ParsedOp pop);
}