IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

simplify usage of ActivityType loader

Browse Source
This commit is contained in:
Jonathan Shook 2021-09-13 09:43:42 -05:00
parent ce695db98b
commit a019258b3d
9 changed files with 142 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityType.java
View File

@ -24,7 +24,6 @@ import io.nosqlbench.engine.api.activityimpl.CoreServices;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.activityimpl.action.CoreActionDispenser;
import io.nosqlbench.engine.api.activityimpl.motor.CoreMotorDispenser;
import io.nosqlbench.nb.api.spi.SimpleServiceLoader;
import java.util.Map;
import java.util.Optional;
@ -39,7 +38,6 @@ import java.util.Optional;
*/
public interface ActivityType<A extends Activity> {
SimpleServiceLoader<ActivityType> FINDER = new SimpleServiceLoader<ActivityType>(ActivityType.class);
/**
* Create an instance of an activity from the activity type.

2
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java
View File

@ -4,11 +4,9 @@ import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.nb.api.spi.SimpleServiceLoader;
public class StandardActivityType<A extends StandardActivity<?,?>> extends SimpleActivity implements ActivityType<A> {
public static SimpleServiceLoader<DriverAdapter> FINDER = new SimpleServiceLoader<>(DriverAdapter.class);
private final DriverAdapter<?,?> adapter;
public StandardActivityType(DriverAdapter<?,?> adapter, ActivityDef activityDef) {

7
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -9,10 +9,7 @@ import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.lifecycle.ScenarioErrorHandler;
import io.nosqlbench.engine.core.lifecycle.ScenarioResult;
import io.nosqlbench.engine.core.lifecycle.ScenariosResults;
import io.nosqlbench.engine.core.lifecycle.ShutdownManager;
import io.nosqlbench.engine.core.lifecycle.*;
import io.nosqlbench.engine.core.logging.LoggerConfig;
import io.nosqlbench.engine.core.metadata.MarkdownDocInfo;
import io.nosqlbench.engine.core.metrics.MetricReporters;
@ -208,7 +205,7 @@ public class NBCLI {
}
if (options.wantsActivityTypes()) {
ActivityType.FINDER.getAllSelectors().forEach(System.out::println);
new ActivityTypeLoader().getAllSelectors().forEach(System.out::println);
System.exit(0);
}

169
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ActivityTypeLoader.java
View File

@ -5,70 +5,143 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType;
import io.nosqlbench.nb.api.NBEnvironment;
import io.nosqlbench.nb.api.config.standard.*;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.spi.SimpleServiceLoader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Optional;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;
public class ActivityTypeLoader {
private static final Logger logger = LogManager.getLogger(ActivityTypeLoader.class);
private static final SimpleServiceLoader<ActivityType> ACTIVITYTYPE_SPI_FINDER = new SimpleServiceLoader<ActivityType>(ActivityType.class);
private static final SimpleServiceLoader<DriverAdapter> DRIVERADAPTER_SPI_FINDER = new SimpleServiceLoader<>(DriverAdapter.class);
private final Set<URL> jarUrls = new HashSet<>();
public static Optional<ActivityType<?>> load(ActivityDef activityDef) {
public ActivityTypeLoader() {
String activityTypeName = activityDef.getParams().getOptionalString("driver", "type").orElse(null);
List<String> libpaths = NBEnvironment.INSTANCE.interpolate(":", "$" + NBEnvironment.NBLIBS);
Set<URL> urlsToAdd = new HashSet<>();
List<String> knownTypes = ActivityType.FINDER.getAllSelectors();
// Infer the type from either alias or yaml if possible (exactly one matches)
if (activityTypeName == null) {
List<String> matching = knownTypes.stream().filter(
n ->
activityDef.getParams().getOptionalString("alias").orElse("").contains(n)
|| activityDef.getParams().getOptionalString("yaml", "workload").orElse("").contains(n)
).collect(Collectors.toList());
if (matching.size() == 1) {
activityTypeName = matching.get(0);
logger.info("param 'type' was inferred as '" + activityTypeName + "' since it was seen in yaml or alias parameter.");
for (String libpaths_entry : libpaths) {
Path libpath = Path.of(libpaths_entry);
if (Files.isDirectory(libpath)) {
urlsToAdd = addLibDir(urlsToAdd, libpath);
} else if (Files.isRegularFile(libpath) && libpath.toString().toLowerCase().endsWith(".zip")) {
urlsToAdd = addZipDir(urlsToAdd, libpath);
} else if (Files.isRegularFile(libpath) && libpath.toString().toLowerCase().endsWith(".jar")) {
urlsToAdd = addJarFile(urlsToAdd, libpath);
}
}
extendClassLoader(urlsToAdd);
}
if (activityTypeName == null) {
String errmsg = "You must provide a driver=<driver> parameter. Valid examples are:\n" +
knownTypes.stream().map(t -> " driver=" + t + "\n").collect(Collectors.joining());
throw new BasicError(errmsg);
private synchronized void extendClassLoader(String... paths) {
Set<URL> urls = new HashSet<>();
for (String path : paths) {
URL url = null;
try {
url = new URL(path);
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
urls.add(url);
}
extendClassLoader(urls);
}
String diagName = activityTypeName;
Optional<ActivityType> ato = ActivityType.FINDER.getOptionally(activityTypeName);
if (ato.isPresent()) {
return Optional.of((ActivityType<?>) ato.get());
}
Optional<DriverAdapter> oda = StandardActivityType.FINDER.getOptionally(activityTypeName);
if (oda.isPresent()) {
DriverAdapter<?, ?> driverAdapter = oda.get();
activityDef.getParams().remove("driver");
if (driverAdapter instanceof NBConfigurable) {
NBConfigModel cfgModel = ((NBConfigurable) driverAdapter).getConfigModel();
cfgModel = cfgModel.add(ACTIVITY_CFG_MODEL);
NBConfiguration cfg = cfgModel.apply(activityDef.getParams());
((NBConfigurable) driverAdapter).applyConfig(cfg);
private synchronized void extendClassLoader(Set<URL> urls) {
Set<URL> newUrls = new HashSet<>();
if (!jarUrls.containsAll(urls)) {
for (URL url : urls) {
if (!jarUrls.contains(url)) {
newUrls.add(url);
jarUrls.add(url);
}
ActivityType activityType = new StandardActivityType<>(driverAdapter, activityDef);
return Optional.of(activityType);
} else {
throw new RuntimeException("Found neither ActivityType named '" + activityTypeName + "' nor DriverAdapter named '" + activityTypeName + "'.");
}
URL[] newUrlAry = newUrls.toArray(new URL[]{});
URLClassLoader ucl = URLClassLoader.newInstance(newUrlAry, Thread.currentThread().getContextClassLoader());
Thread.currentThread().setContextClassLoader(ucl);
logger.debug("Extended class loader layering with " + newUrls);
} else {
logger.debug("All URLs specified were already in a class loader.");
}
}
private Set<URL> addJarFile(Set<URL> urls, Path libpath) {
try {
urls.add(libpath.toUri().toURL());
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
return urls;
}
private Set<URL> addZipDir(Set<URL> urlsToAdd, Path libpath) {
return urlsToAdd;
}
private Set<URL> addLibDir(Set<URL> urlsToAdd, Path libpath) {
Set<URL> urls = NBIO.local()
.prefix(libpath.toString())
.extension(".jar")
.list().stream().map(Content::getURL)
.collect(Collectors.toSet());
urlsToAdd.addAll(urls);
return urlsToAdd;
}
public Optional<ActivityType> load(ActivityDef activityDef) {
final String driverName = activityDef.getParams()
.getOptionalString("driver", "type")
.orElseThrow(() -> new BasicError("The parameter 'driver=' is required."));
activityDef.getParams()
.getOptionalString("jar")
.map(jar -> {
Set<URL> urls = NBIO.local().search(jar)
.list()
.stream().map(Content::getURL)
.collect(Collectors.toSet());
return urls;
})
.ifPresent(this::extendClassLoader);
return this.getDriverAdapter(driverName,activityDef)
.or(() -> ACTIVITYTYPE_SPI_FINDER.getOptionally(driverName));
}
private Optional<ActivityType> getDriverAdapter(String activityTypeName, ActivityDef activityDef) {
Optional<DriverAdapter> oda = DRIVERADAPTER_SPI_FINDER.getOptionally(activityTypeName);
if (oda.isPresent()) {
DriverAdapter<?, ?> driverAdapter = oda.get();
activityDef.getParams().remove("driver");
if (driverAdapter instanceof NBConfigurable) {
NBConfigModel cfgModel = ((NBConfigurable) driverAdapter).getConfigModel();
cfgModel = cfgModel.add(ACTIVITY_CFG_MODEL);
NBConfiguration cfg = cfgModel.apply(activityDef.getParams());
((NBConfigurable) driverAdapter).applyConfig(cfg);
}
ActivityType activityType = new StandardActivityType<>(driverAdapter, activityDef);
return Optional.of(activityType);
} else {
return Optional.empty();
}
}
private static final NBConfigModel ACTIVITY_CFG_MODEL = ConfigModel.of(Activity.class)
@ -80,4 +153,12 @@ public class ActivityTypeLoader {
.add(Param.optional("tags"))
.asReadOnly();
public Set<String> getAllSelectors() {
List<String> allSelectors = ACTIVITYTYPE_SPI_FINDER.getAllSelectors();
List<String> allDrivers = DRIVERADAPTER_SPI_FINDER.getAllSelectors();
Set<String> all = new HashSet<>();
all.addAll(allSelectors);
all.addAll(allDrivers);
return all;
}
}

2
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioController.java
View File

@ -279,7 +279,7 @@ public class ScenarioController {
if (executor == null && createIfMissing) {
ActivityType<?> activityType = ActivityTypeLoader.load(activityDef).orElseThrow(
ActivityType<?> activityType = new ActivityTypeLoader().load(activityDef).orElseThrow(
() -> new RuntimeException("Could not load Driver for " + activityDef + "'")
);

7
engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownDocInfo.java
View File

@ -1,9 +1,12 @@
package io.nosqlbench.engine.core.metadata;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -44,7 +47,9 @@ public class MarkdownDocInfo {
}
public String forActivityInstance(String s) {
ActivityType activityType = ActivityType.FINDER.getOrThrow(s);
ActivityType activityType = new ActivityTypeLoader().load(ActivityDef.parseActivityDef("driver="+s)).orElseThrow(
() -> new BasicError("Unable to find driver for '" + s + "'")
);
return forResourceMarkdown(activityType.getClass().getAnnotation(Service.class)
.selector() + ".md", "docs/");
}

3
engine-core/src/main/java/io/nosqlbench/engine/core/script/MetricsMapper.java
View File

@ -19,6 +19,7 @@ import io.nosqlbench.engine.api.activityapi.core.Activity;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader;
import io.nosqlbench.engine.core.metrics.PolyglotMetricRegistryBindings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -62,7 +63,7 @@ public class MetricsMapper {
ActivityDef activityDef = ActivityDef.parseActivityDef(activitySpec);
logger.info("introspecting metric names for " + activitySpec);
Optional<ActivityType> activityType = ActivityType.FINDER.get(activityDef.getActivityType());
Optional<ActivityType> activityType = new ActivityTypeLoader().load(activityDef);
if (!activityType.isPresent()) {
throw new RuntimeException("Activity type '" + activityDef.getActivityType() + "' does not exist in this runtime.");

7
engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java
View File

@ -13,6 +13,7 @@ import io.nosqlbench.engine.api.activityimpl.input.AtomicInput;
import io.nosqlbench.engine.api.activityimpl.motor.CoreMotor;
import io.nosqlbench.engine.api.activityimpl.motor.CoreMotorDispenser;
import io.nosqlbench.engine.core.lifecycle.ActivityExecutor;
import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.junit.jupiter.api.Test;
@ -41,7 +42,7 @@ public class ActivityExecutorTest {
@Test
public void testRestart() {
ActivityDef ad = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;initdelay=5000;");
Optional<ActivityType> activityType = ActivityType.FINDER.get(ad.getActivityType());
Optional<ActivityType> activityType = new ActivityTypeLoader().load(ad);
Activity a = new DelayedInitActivity(ad);
InputDispenser idisp = new CoreInputDispenser(a);
ActionDispenser adisp = new CoreActionDispenser(a);
@ -65,7 +66,7 @@ public class ActivityExecutorTest {
@Test
public void testDelayedStartSanity() {
ActivityDef ad = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;initdelay=5000;");
Optional<ActivityType> activityType = ActivityType.FINDER.get(ad.getActivityType());
Optional<ActivityType> activityType = new ActivityTypeLoader().load(ad);
Activity a = new DelayedInitActivity(ad);
InputDispenser idisp = new CoreInputDispenser(a);
ActionDispenser adisp = new CoreActionDispenser(a);
@ -87,7 +88,7 @@ public class ActivityExecutorTest {
@Test
public void testNewActivityExecutor() {
ActivityDef ad = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;");
Optional<ActivityType> activityType = ActivityType.FINDER.get(ad.getActivityType());
Optional<ActivityType> activityType = new ActivityTypeLoader().load(ad);
Input longSupplier = new AtomicInput(ad);
MotorDispenser<?> cmf = getActivityMotorFactory(
ad, motorActionDelay(999), longSupplier

3
nb-api/src/main/java/io/nosqlbench/nb/api/spi/SimpleServiceLoader.java
View File

@ -57,7 +57,8 @@ public class SimpleServiceLoader<T> {
/**
* Load the service providers which are annotated with {@link Service} and selector names.
*
* @param includes If provided, a list of patterns which are used to include named services based on the selector name from the
* @param includes If provided, a list of patterns which are used to include
* named services based on the selector name from the
* {@link Service} annotation.
* @return A map of providers of T
*/