From a019258b3dd89b087fafcb375fe4a9cd78cd90c5 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 13 Sep 2021 09:43:42 -0500 Subject: [PATCH] simplify usage of ActivityType loader --- .../api/activityapi/core/ActivityType.java | 2 - .../uniform/StandardActivityType.java | 2 - .../java/io/nosqlbench/engine/cli/NBCLI.java | 7 +- .../core/lifecycle/ActivityTypeLoader.java | 169 +++++++++++++----- .../core/lifecycle/ScenarioController.java | 2 +- .../engine/core/metadata/MarkdownDocInfo.java | 7 +- .../engine/core/script/MetricsMapper.java | 3 +- .../engine/core/ActivityExecutorTest.java | 7 +- .../nb/api/spi/SimpleServiceLoader.java | 3 +- 9 files changed, 142 insertions(+), 60 deletions(-) diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityType.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityType.java index 08dfb5760..9a55a0b42 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityType.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/ActivityType.java @@ -24,7 +24,6 @@ import io.nosqlbench.engine.api.activityimpl.CoreServices; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; import io.nosqlbench.engine.api.activityimpl.action.CoreActionDispenser; import io.nosqlbench.engine.api.activityimpl.motor.CoreMotorDispenser; -import io.nosqlbench.nb.api.spi.SimpleServiceLoader; import java.util.Map; import java.util.Optional; @@ -39,7 +38,6 @@ import java.util.Optional; */ public interface ActivityType { - SimpleServiceLoader FINDER = new SimpleServiceLoader(ActivityType.class); /** * Create an instance of an activity from the activity type. diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java index 582fec6d6..a311f29eb 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java @@ -4,11 +4,9 @@ import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; import io.nosqlbench.engine.api.activityapi.core.ActivityType; import io.nosqlbench.engine.api.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityimpl.SimpleActivity; -import io.nosqlbench.nb.api.spi.SimpleServiceLoader; public class StandardActivityType> extends SimpleActivity implements ActivityType { - public static SimpleServiceLoader FINDER = new SimpleServiceLoader<>(DriverAdapter.class); private final DriverAdapter adapter; public StandardActivityType(DriverAdapter adapter, ActivityDef activityDef) { diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 10422384f..1c00ae202 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -9,10 +9,7 @@ import io.nosqlbench.engine.api.activityapi.output.OutputType; import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader; import io.nosqlbench.engine.api.metrics.ActivityMetrics; import io.nosqlbench.engine.core.annotation.Annotators; -import io.nosqlbench.engine.core.lifecycle.ScenarioErrorHandler; -import io.nosqlbench.engine.core.lifecycle.ScenarioResult; -import io.nosqlbench.engine.core.lifecycle.ScenariosResults; -import io.nosqlbench.engine.core.lifecycle.ShutdownManager; +import io.nosqlbench.engine.core.lifecycle.*; import io.nosqlbench.engine.core.logging.LoggerConfig; import io.nosqlbench.engine.core.metadata.MarkdownDocInfo; import io.nosqlbench.engine.core.metrics.MetricReporters; @@ -208,7 +205,7 @@ public class NBCLI { } if (options.wantsActivityTypes()) { - ActivityType.FINDER.getAllSelectors().forEach(System.out::println); + new ActivityTypeLoader().getAllSelectors().forEach(System.out::println); System.exit(0); } diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ActivityTypeLoader.java b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ActivityTypeLoader.java index 032aea1ec..558978c7c 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ActivityTypeLoader.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ActivityTypeLoader.java @@ -5,70 +5,143 @@ import io.nosqlbench.engine.api.activityapi.core.ActivityType; import io.nosqlbench.engine.api.activityimpl.ActivityDef; import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.engine.api.activityimpl.uniform.StandardActivityType; +import io.nosqlbench.nb.api.NBEnvironment; import io.nosqlbench.nb.api.config.standard.*; +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.content.NBIO; import io.nosqlbench.nb.api.errors.BasicError; +import io.nosqlbench.nb.api.spi.SimpleServiceLoader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.List; -import java.util.Optional; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; import java.util.stream.Collectors; public class ActivityTypeLoader { + private static final Logger logger = LogManager.getLogger(ActivityTypeLoader.class); + private static final SimpleServiceLoader ACTIVITYTYPE_SPI_FINDER = new SimpleServiceLoader(ActivityType.class); + private static final SimpleServiceLoader DRIVERADAPTER_SPI_FINDER = new SimpleServiceLoader<>(DriverAdapter.class); + private final Set jarUrls = new HashSet<>(); - public static Optional> load(ActivityDef activityDef) { + public ActivityTypeLoader() { - String activityTypeName = activityDef.getParams().getOptionalString("driver", "type").orElse(null); + List libpaths = NBEnvironment.INSTANCE.interpolate(":", "$" + NBEnvironment.NBLIBS); + Set urlsToAdd = new HashSet<>(); - List knownTypes = ActivityType.FINDER.getAllSelectors(); - - // Infer the type from either alias or yaml if possible (exactly one matches) - if (activityTypeName == null) { - List matching = knownTypes.stream().filter( - n -> - activityDef.getParams().getOptionalString("alias").orElse("").contains(n) - || activityDef.getParams().getOptionalString("yaml", "workload").orElse("").contains(n) - ).collect(Collectors.toList()); - if (matching.size() == 1) { - activityTypeName = matching.get(0); - logger.info("param 'type' was inferred as '" + activityTypeName + "' since it was seen in yaml or alias parameter."); + for (String libpaths_entry : libpaths) { + Path libpath = Path.of(libpaths_entry); + if (Files.isDirectory(libpath)) { + urlsToAdd = addLibDir(urlsToAdd, libpath); + } else if (Files.isRegularFile(libpath) && libpath.toString().toLowerCase().endsWith(".zip")) { + urlsToAdd = addZipDir(urlsToAdd, libpath); + } else if (Files.isRegularFile(libpath) && libpath.toString().toLowerCase().endsWith(".jar")) { + urlsToAdd = addJarFile(urlsToAdd, libpath); } } + extendClassLoader(urlsToAdd); + } - if (activityTypeName == null) { - String errmsg = "You must provide a driver= parameter. Valid examples are:\n" + - knownTypes.stream().map(t -> " driver=" + t + "\n").collect(Collectors.joining()); - throw new BasicError(errmsg); + private synchronized void extendClassLoader(String... paths) { + Set urls = new HashSet<>(); + for (String path : paths) { + URL url = null; + try { + url = new URL(path); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + urls.add(url); } + extendClassLoader(urls); + } - String diagName = activityTypeName; - - - Optional ato = ActivityType.FINDER.getOptionally(activityTypeName); - if (ato.isPresent()) { - return Optional.of((ActivityType) ato.get()); - } - - Optional oda = StandardActivityType.FINDER.getOptionally(activityTypeName); - - if (oda.isPresent()) { - DriverAdapter driverAdapter = oda.get(); - - activityDef.getParams().remove("driver"); - if (driverAdapter instanceof NBConfigurable) { - NBConfigModel cfgModel = ((NBConfigurable) driverAdapter).getConfigModel(); - cfgModel = cfgModel.add(ACTIVITY_CFG_MODEL); - NBConfiguration cfg = cfgModel.apply(activityDef.getParams()); - ((NBConfigurable) driverAdapter).applyConfig(cfg); + private synchronized void extendClassLoader(Set urls) { + Set newUrls = new HashSet<>(); + if (!jarUrls.containsAll(urls)) { + for (URL url : urls) { + if (!jarUrls.contains(url)) { + newUrls.add(url); + jarUrls.add(url); } - ActivityType activityType = new StandardActivityType<>(driverAdapter, activityDef); - return Optional.of(activityType); - - } else { - throw new RuntimeException("Found neither ActivityType named '" + activityTypeName + "' nor DriverAdapter named '" + activityTypeName + "'."); } + URL[] newUrlAry = newUrls.toArray(new URL[]{}); + URLClassLoader ucl = URLClassLoader.newInstance(newUrlAry, Thread.currentThread().getContextClassLoader()); + Thread.currentThread().setContextClassLoader(ucl); + logger.debug("Extended class loader layering with " + newUrls); + } else { + logger.debug("All URLs specified were already in a class loader."); + } + } + private Set addJarFile(Set urls, Path libpath) { + try { + urls.add(libpath.toUri().toURL()); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + return urls; + } + + private Set addZipDir(Set urlsToAdd, Path libpath) { + return urlsToAdd; + } + + private Set addLibDir(Set urlsToAdd, Path libpath) { + Set urls = NBIO.local() + .prefix(libpath.toString()) + .extension(".jar") + .list().stream().map(Content::getURL) + .collect(Collectors.toSet()); + urlsToAdd.addAll(urls); + return urlsToAdd; + } + + public Optional load(ActivityDef activityDef) { + + final String driverName = activityDef.getParams() + .getOptionalString("driver", "type") + .orElseThrow(() -> new BasicError("The parameter 'driver=' is required.")); + + activityDef.getParams() + .getOptionalString("jar") + .map(jar -> { + Set urls = NBIO.local().search(jar) + .list() + .stream().map(Content::getURL) + .collect(Collectors.toSet()); + return urls; + }) + .ifPresent(this::extendClassLoader); + + return this.getDriverAdapter(driverName,activityDef) + .or(() -> ACTIVITYTYPE_SPI_FINDER.getOptionally(driverName)); + + } + + private Optional getDriverAdapter(String activityTypeName, ActivityDef activityDef) { + Optional oda = DRIVERADAPTER_SPI_FINDER.getOptionally(activityTypeName); + + if (oda.isPresent()) { + DriverAdapter driverAdapter = oda.get(); + + activityDef.getParams().remove("driver"); + if (driverAdapter instanceof NBConfigurable) { + NBConfigModel cfgModel = ((NBConfigurable) driverAdapter).getConfigModel(); + cfgModel = cfgModel.add(ACTIVITY_CFG_MODEL); + NBConfiguration cfg = cfgModel.apply(activityDef.getParams()); + ((NBConfigurable) driverAdapter).applyConfig(cfg); + } + ActivityType activityType = new StandardActivityType<>(driverAdapter, activityDef); + return Optional.of(activityType); + } else { + return Optional.empty(); + } } private static final NBConfigModel ACTIVITY_CFG_MODEL = ConfigModel.of(Activity.class) @@ -80,4 +153,12 @@ public class ActivityTypeLoader { .add(Param.optional("tags")) .asReadOnly(); + public Set getAllSelectors() { + List allSelectors = ACTIVITYTYPE_SPI_FINDER.getAllSelectors(); + List allDrivers = DRIVERADAPTER_SPI_FINDER.getAllSelectors(); + Set all = new HashSet<>(); + all.addAll(allSelectors); + all.addAll(allDrivers); + return all; + } } diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioController.java b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioController.java index 22e14c301..21e62750f 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioController.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/ScenarioController.java @@ -279,7 +279,7 @@ public class ScenarioController { if (executor == null && createIfMissing) { - ActivityType activityType = ActivityTypeLoader.load(activityDef).orElseThrow( + ActivityType activityType = new ActivityTypeLoader().load(activityDef).orElseThrow( () -> new RuntimeException("Could not load Driver for " + activityDef + "'") ); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownDocInfo.java b/engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownDocInfo.java index db9e9c644..74e61de64 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownDocInfo.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownDocInfo.java @@ -1,9 +1,12 @@ package io.nosqlbench.engine.core.metadata; import io.nosqlbench.engine.api.activityapi.core.ActivityType; +import io.nosqlbench.engine.api.activityimpl.ActivityDef; +import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.nb.api.content.Content; import io.nosqlbench.nb.api.content.NBIO; +import io.nosqlbench.nb.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -44,7 +47,9 @@ public class MarkdownDocInfo { } public String forActivityInstance(String s) { - ActivityType activityType = ActivityType.FINDER.getOrThrow(s); + ActivityType activityType = new ActivityTypeLoader().load(ActivityDef.parseActivityDef("driver="+s)).orElseThrow( + () -> new BasicError("Unable to find driver for '" + s + "'") + ); return forResourceMarkdown(activityType.getClass().getAnnotation(Service.class) .selector() + ".md", "docs/"); } diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/script/MetricsMapper.java b/engine-core/src/main/java/io/nosqlbench/engine/core/script/MetricsMapper.java index b70f7f390..6ce952031 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/script/MetricsMapper.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/script/MetricsMapper.java @@ -19,6 +19,7 @@ import io.nosqlbench.engine.api.activityapi.core.Activity; import io.nosqlbench.engine.api.activityapi.core.ActivityType; import io.nosqlbench.engine.api.activityimpl.ActivityDef; import io.nosqlbench.engine.api.metrics.ActivityMetrics; +import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader; import io.nosqlbench.engine.core.metrics.PolyglotMetricRegistryBindings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,7 +63,7 @@ public class MetricsMapper { ActivityDef activityDef = ActivityDef.parseActivityDef(activitySpec); logger.info("introspecting metric names for " + activitySpec); - Optional activityType = ActivityType.FINDER.get(activityDef.getActivityType()); + Optional activityType = new ActivityTypeLoader().load(activityDef); if (!activityType.isPresent()) { throw new RuntimeException("Activity type '" + activityDef.getActivityType() + "' does not exist in this runtime."); diff --git a/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java b/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java index f15ab5a33..0fa0ea0e3 100644 --- a/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java +++ b/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java @@ -13,6 +13,7 @@ import io.nosqlbench.engine.api.activityimpl.input.AtomicInput; import io.nosqlbench.engine.api.activityimpl.motor.CoreMotor; import io.nosqlbench.engine.api.activityimpl.motor.CoreMotorDispenser; import io.nosqlbench.engine.core.lifecycle.ActivityExecutor; +import io.nosqlbench.engine.core.lifecycle.ActivityTypeLoader; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.junit.jupiter.api.Test; @@ -41,7 +42,7 @@ public class ActivityExecutorTest { @Test public void testRestart() { ActivityDef ad = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;initdelay=5000;"); - Optional activityType = ActivityType.FINDER.get(ad.getActivityType()); + Optional activityType = new ActivityTypeLoader().load(ad); Activity a = new DelayedInitActivity(ad); InputDispenser idisp = new CoreInputDispenser(a); ActionDispenser adisp = new CoreActionDispenser(a); @@ -65,7 +66,7 @@ public class ActivityExecutorTest { @Test public void testDelayedStartSanity() { ActivityDef ad = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;initdelay=5000;"); - Optional activityType = ActivityType.FINDER.get(ad.getActivityType()); + Optional activityType = new ActivityTypeLoader().load(ad); Activity a = new DelayedInitActivity(ad); InputDispenser idisp = new CoreInputDispenser(a); ActionDispenser adisp = new CoreActionDispenser(a); @@ -87,7 +88,7 @@ public class ActivityExecutorTest { @Test public void testNewActivityExecutor() { ActivityDef ad = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;"); - Optional activityType = ActivityType.FINDER.get(ad.getActivityType()); + Optional activityType = new ActivityTypeLoader().load(ad); Input longSupplier = new AtomicInput(ad); MotorDispenser cmf = getActivityMotorFactory( ad, motorActionDelay(999), longSupplier diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/spi/SimpleServiceLoader.java b/nb-api/src/main/java/io/nosqlbench/nb/api/spi/SimpleServiceLoader.java index 89941a8e5..fcfed676f 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/spi/SimpleServiceLoader.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/spi/SimpleServiceLoader.java @@ -57,7 +57,8 @@ public class SimpleServiceLoader { /** * Load the service providers which are annotated with {@link Service} and selector names. * - * @param includes If provided, a list of patterns which are used to include named services based on the selector name from the + * @param includes If provided, a list of patterns which are used to include + * named services based on the selector name from the * {@link Service} annotation. * @return A map of providers of T */