From aef0179bbe478fcf073af7e640bce9f2eca11d24 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Thu, 6 Aug 2020 09:52:02 -0500 Subject: [PATCH] add engine-rest --- .../engine/core/ScenarioLogger.java | 2 +- .../engine/core/ScenarioResult.java | 1 - .../engine/core/ScenariosResults.java | 6 +- .../engine/core/script/ScenariosExecutor.java | 8 +- engine-rest/pom.xml | 49 ++++ .../resources/ScenarioExecutorService.java | 210 ++++++++++++++++++ .../resources}/ScenarioTemplateService.java | 2 +- .../engine/rest/transfertypes/ResultInfo.java | 30 +++ .../transfertypes/RunScenarioRequest.java | 70 ++++++ .../rest/transfertypes/ScenarioInfo.java | 35 +++ nb/pom.xml | 5 + pom.xml | 7 +- 12 files changed, 413 insertions(+), 12 deletions(-) create mode 100644 engine-rest/pom.xml create mode 100644 engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorService.java rename {engine-core/src/main/java/io/nosqlbench/engine/services => engine-rest/src/main/java/io/nosqlbench/engine/rest/resources}/ScenarioTemplateService.java (91%) create mode 100644 engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultInfo.java create mode 100644 engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/RunScenarioRequest.java create mode 100644 engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ScenarioInfo.java diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java index 8cd68a542..1c1455851 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java @@ -87,7 +87,7 @@ public class ScenarioLogger { ple.setContext(loggerContext); ple.start(); - String scenarioLog = loggerDir.getPath() + File.separator + scenario.getName()+".log"; + String scenarioLog = loggerDir.getPath() + File.separator + scenario.getScenarioName()+".log"; scenarioLog = scenarioLog.replaceAll("\\s","_"); FileAppender fileAppender = new FileAppender(); fileAppender.setFile(scenarioLog); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java index d1fa12fe8..8d2545bf8 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java @@ -33,7 +33,6 @@ public class ScenarioResult { private Exception exception; private String iolog; - private String report; public ScenarioResult(String iolog) { this.iolog = iolog; diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenariosResults.java b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenariosResults.java index 3ef353b22..85ebacb85 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/ScenariosResults.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/ScenariosResults.java @@ -29,8 +29,8 @@ public class ScenariosResults { private static final Logger logger = LoggerFactory.getLogger(ScenariosResults.class); - private String scenariosExecutorName; - private Map scenarioResultMap = new LinkedHashMap<>(); + private final String scenariosExecutorName; + private final Map scenarioResultMap = new LinkedHashMap<>(); public ScenariosResults(ScenariosExecutor scenariosExecutor) { @@ -68,7 +68,7 @@ public class ScenariosResults { if (oresult != null) { oresult.reportToLog(); } else { - logger.error(scenario.getName() + ": incomplete (missing result)"); + logger.error(scenario.getScenarioName() + ": incomplete (missing result)"); } } } diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScenariosExecutor.java b/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScenariosExecutor.java index 02595951f..5dbc526a5 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScenariosExecutor.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/script/ScenariosExecutor.java @@ -40,7 +40,7 @@ public class ScenariosExecutor { } public ScenariosExecutor(String name, int threads) { - executor = new ThreadPoolExecutor(1, 1, + executor = new ThreadPoolExecutor(1, threads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new IndexedThreadFactory("scenarios", new ScenarioExceptionHandler(this))); @@ -53,8 +53,8 @@ public class ScenariosExecutor { public synchronized void execute(Scenario scenario, ScenarioLogger scenarioLogger) { scenario.setScenarioLogger(scenarioLogger); - if (submitted.get(scenario.getName()) != null) { - throw new BasicError("Scenario " + scenario.getName() + " is already defined. Remove it first to reuse the name."); + if (submitted.get(scenario.getScenarioName()) != null) { + throw new BasicError("Scenario " + scenario.getScenarioName() + " is already defined. Remove it first to reuse the name."); } Future future = executor.submit(scenario); SubmittedScenario s = new SubmittedScenario(scenario, future); @@ -229,7 +229,7 @@ public class ScenariosExecutor { } public String getName() { - return scenario.getName(); + return scenario.getScenarioName(); } } diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml new file mode 100644 index 000000000..ac0da5619 --- /dev/null +++ b/engine-rest/pom.xml @@ -0,0 +1,49 @@ + + 4.0.0 + + + mvn-defaults + io.nosqlbench + 3.12.134-SNAPSHOT + ../mvn-defaults + + + engine-rest + jar + ${project.artifactId} + REST services for nosqlbench + + + UTF-8 + nosqlbench REST Services + + + + + + io.nosqlbench + engine-cli + 3.12.134-SNAPSHOT + + + + junit + junit + test + + + + org.assertj + assertj-core + test + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + + diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorService.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorService.java new file mode 100644 index 000000000..15d5bbedf --- /dev/null +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorService.java @@ -0,0 +1,210 @@ +package io.nosqlbench.engine.rest.resources; + +import io.nosqlbench.docsys.api.WebServiceObject; +import io.nosqlbench.engine.cli.BasicScriptBuffer; +import io.nosqlbench.engine.cli.Cmd; +import io.nosqlbench.engine.cli.NBCLICommandParser; +import io.nosqlbench.engine.cli.ScriptBuffer; +import io.nosqlbench.engine.core.ScenarioResult; +import io.nosqlbench.engine.core.script.Scenario; +import io.nosqlbench.engine.core.script.ScenariosExecutor; +import io.nosqlbench.engine.rest.transfertypes.RunScenarioRequest; +import io.nosqlbench.engine.rest.transfertypes.ScenarioInfo; +import io.nosqlbench.engine.rest.transfertypes.ResultInfo; +import io.nosqlbench.nb.annotations.Service; + +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.*; + +@Service(WebServiceObject.class) +@Singleton +@Path("/services/executor/") +public class ScenarioExecutorService implements WebServiceObject { + + private ScenariosExecutor executor = new ScenariosExecutor("executor-service", 1); + + @POST + @Path("cli") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public synchronized Response invokeCommand(RunScenarioRequest rq) { + + String name = rq.getScenarioName(); + if (name.equals("auto")) { + rq.setScenarioName("scenario"+String.valueOf(System.currentTimeMillis())); + } + + // First, virtualize files provided + storeFiles(rq); + + LinkedList cmdList = new LinkedList<>(); + LinkedList args = new LinkedList<>(rq.getCommands()); + + for (String arg : args) { + if (arg.startsWith("-")) { + throw new RuntimeException("Only commands (verbs and params) can be used here"); + } + } + + args = substituteFilenames(rq, args); + NBCLICommandParser.parse(args, cmdList); + ScriptBuffer buffer = new BasicScriptBuffer(); + buffer.add(cmdList.toArray(new Cmd[0])); + + Scenario scenario = new Scenario( + rq.getScenarioName(), + "", + Scenario.Engine.Graalvm, + "disabled", + false, + true, + false + ); + scenario.addScriptText(buffer.getParsedScript()); + + executor.execute(scenario); + + return Response.created(UriBuilder.fromResource(ScenarioExecutorService.class).path( + "scenario/" + rq.getScenarioName()).build()).build(); + + } + + private LinkedList substituteFilenames(RunScenarioRequest rq, LinkedList args) { + LinkedList newargs = new LinkedList<>(); + for (String arg : args) { + for (String s : rq.getFilemap().keySet()) { + arg = arg.replaceAll(s,rq.getFilemap().get(s)); + newargs.add(arg); + } + } + return newargs; + } + + private void storeFiles(RunScenarioRequest rq) { + Map filemap = rq.getFilemap(); + + for (String filename : filemap.keySet()) { + try { + Paths.get(rq.getBasedir(),rq.getScenarioName()); + + Files.createDirectories( + Paths.get(rq.getBasedir(),rq.getScenarioName()), + PosixFilePermissions.asFileAttribute( + PosixFilePermissions.fromString("rwxr-x---") + )); + java.nio.file.Path tmpyaml = Files.createTempFile( + Paths.get("/tmp/nosqlbench"), + rq.getScenarioName(), + filename + ); +// // TODO: Find a better way to do this, like scoping resources to executor +// tmpyaml.toFile().deleteOnExit(); + + Files.write( + tmpyaml, + filemap.get(filename).getBytes(StandardCharsets.UTF_8) + ); + rq.getFilemap().put(filename, tmpyaml.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + +// /** +// * Run a single-activity scenario +// * +// * @param scenarioName +// * The name to install in the executor +// * @param params +// * The params for the activity +// * +// * @return +// */ +// @POST +// @Path("scenario/{scenarioName}") +// @Consumes(MediaType.APPLICATION_JSON) +// @Produces(MediaType.APPLICATION_JSON) +// public synchronized Response invokeScenario( +// @PathParam("scenarioName") String scenarioName, +// Map params) { +// Scenario scenario = null; +// Optional pendingScenario = executor.getPendingScenario(scenarioName); +// if (pendingScenario.isPresent()) { +// scenario = pendingScenario.orElseThrow(); +// } else { +// scenario = new Scenario(scenarioName, Scenario.Engine.Graalvm); +// } +// if (params.containsKey("yamldoc")) { +// try { +// java.nio.file.Path tmpyaml = Files.createTempFile(Paths.get("/tmp"), scenarioName, ".yaml"); +// // TODO: Find a better way to do this, like scoping resources to executor +// tmpyaml.toFile().deleteOnExit(); +// Files.write(tmpyaml, params.get("yamldoc").getBytes(StandardCharsets.UTF_8)); +// params.remove("yamldoc"); +// params.put("yaml", tmpyaml.toString()); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// } +// scenario.getScenarioController().apply(params); +// URI scenarioUri = UriBuilder.fromResource(ScenarioExecutorService.class) +// .build(scenarioName); +// return Response.created(scenarioUri).build(); +// } + + @GET + @Path("scenario/{scenarioName}") + @Produces(MediaType.APPLICATION_JSON) + public synchronized ScenarioInfo getScenario(@PathParam("scenarioName") String scenarioName) { + Optional pendingScenario = executor.getPendingScenario(scenarioName); + if (pendingScenario.isPresent()) { + return new ScenarioInfo(pendingScenario.get()); + } else { + throw new RuntimeException("Scenario name '" + scenarioName + "' not found."); + } + } + + @GET + @Path("scenarios") + @Produces(MediaType.APPLICATION_JSON) + public synchronized List getScenarios() { + List scenarioInfos = new ArrayList<>(); + List pendingScenarios = executor.getPendingScenarios(); + for (String pendingName : pendingScenarios) { + Optional pendingScenario = executor.getPendingScenario(pendingName); + pendingScenario.ifPresent(scenario -> scenarioInfos.add(new ScenarioInfo(scenario))); + } + return scenarioInfos; + } + + @GET + @Path("result/{scenarioName}") + @Produces(MediaType.APPLICATION_JSON) + public synchronized ResultInfo getResult(@PathParam("scenarioName") String scenarioName) { + return new ResultInfo(scenarioName, executor.getPendingResult(scenarioName).orElse(null)); + } + + @GET + @Path("results") + @Produces(MediaType.APPLICATION_JSON) + public synchronized List getResults() { + List results = new ArrayList<>(); + List pendingScenarios = executor.getPendingScenarios(); + for (String pendingScenario : pendingScenarios) { + Optional pendingResult = executor.getPendingResult(pendingScenario); + results.add(new ResultInfo(pendingScenario, pendingResult.orElse(null))); + } + return results; + } + +} diff --git a/engine-core/src/main/java/io/nosqlbench/engine/services/ScenarioTemplateService.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioTemplateService.java similarity index 91% rename from engine-core/src/main/java/io/nosqlbench/engine/services/ScenarioTemplateService.java rename to engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioTemplateService.java index 7eae7890b..f7fbe205e 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/services/ScenarioTemplateService.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioTemplateService.java @@ -1,4 +1,4 @@ -package io.nosqlbench.engine.services; +package io.nosqlbench.engine.rest.resources; import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.nb.annotations.Service; diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultInfo.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultInfo.java new file mode 100644 index 000000000..e8cf08970 --- /dev/null +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultInfo.java @@ -0,0 +1,30 @@ +package io.nosqlbench.engine.rest.transfertypes; + +import io.nosqlbench.engine.core.ScenarioResult; + +public class ResultInfo { + private final String scenarioName; + private final ScenarioResult result; + + public ResultInfo(String scenarioName, ScenarioResult result) { + this.scenarioName = scenarioName; + this.result = result; + } + + public String getScenarioName() { + return scenarioName; + } + + public boolean isComplete() { + return result != null; + } + + public boolean isErrored() { + return (result != null && result.getException().isPresent()); + } + + public String getIOLog() { + return result.getIOLog(); + } + +} diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/RunScenarioRequest.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/RunScenarioRequest.java new file mode 100644 index 000000000..ed045355f --- /dev/null +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/RunScenarioRequest.java @@ -0,0 +1,70 @@ +package io.nosqlbench.engine.rest.transfertypes; + +import java.util.List; +import java.util.Map; + +/** + * A CliRequest is what the user sends when they want to invoke NoSQLBench via web request in the + * same way they may on the command line. + * + * 
{@code
+ *  {
+ *    "name" : "auto",
+ *    "basedir" : "/tmp/nosqlbench",
+ *    "filemap" : {
+ *        "file1.yaml": "contents of file1"
+ *    },
+ *    "commands": [
+ *      "run", "workload=file1.yaml", "driver=stdout", "cycles=10M", "cyclerate=100"
+ *    ]
+ *  }
+ * }
+ */ +public class RunScenarioRequest { + + private List commands; + private Map filemap; + private String stdout; + private String scenarioName = "auto"; + private String basedir = "/tmp/nosqlbench"; + + public void setScenarioName(String scenarioName) { + this.scenarioName = scenarioName; + } + + public String getScenarioName() { + return scenarioName; + } + + public void setCommands(List commands) { + this.commands = commands; + } + + public void setFileMap(Map filemap) { + this.filemap = filemap; + } + + public void setStdout(String stdout) { + this.stdout = stdout; + } + + public List getCommands() { + return commands; + } + + public Map getFilemap() { + return filemap; + } + + public String getStdout() { + return stdout; + } + + public void setBasedir(String basedir) { + this.basedir = basedir; + } + + public String getBasedir() { + return basedir; + } +} diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ScenarioInfo.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ScenarioInfo.java new file mode 100644 index 000000000..31c274ca0 --- /dev/null +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ScenarioInfo.java @@ -0,0 +1,35 @@ +package io.nosqlbench.engine.rest.transfertypes; + +import io.nosqlbench.engine.api.activityapi.core.ProgressMeter; +import io.nosqlbench.engine.core.script.Scenario; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class ScenarioInfo { + private final Scenario scenario; + + public ScenarioInfo(Scenario scenario) { + this.scenario = scenario; + } + + public String getScenarioName() { + return scenario.getScenarioName(); + } + + public Map getProgress() { + Map progress = new HashMap<>(); + + Collection progressMeters = + scenario.getScenarioController().getProgressMeters(); + for (ProgressMeter meter : progressMeters) { + String activityName = meter.getProgressName(); + String activityProgress = meter.getProgressDetails(); + if (activityName!=null && activityProgress!=null) { + progress.put(activityName, activityProgress); + } + } + return progress; + } +} diff --git a/nb/pom.xml b/nb/pom.xml index e84b1f579..3314d3284 100644 --- a/nb/pom.xml +++ b/nb/pom.xml @@ -32,6 +32,11 @@ driver-kafka 3.12.134-SNAPSHOT + + io.nosqlbench + engine-rest + 3.12.134-SNAPSHOT + io.nosqlbench engine-cli diff --git a/pom.xml b/pom.xml index 8549a1608..8ee07819c 100644 --- a/pom.xml +++ b/pom.xml @@ -26,16 +26,19 @@ nb-api nb-annotations - + engine-api engine-core - engine-cli engine-extensions engine-docker engine-docs + engine-cli + engine-rest nb + + driver-diag driver-stdout driver-tcp