add engine-rest

engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioLogger.java

@@ -87,7 +87,7 @@ public class ScenarioLogger {
         ple.setContext(loggerContext);
         ple.start();
 
-        String scenarioLog = loggerDir.getPath() + File.separator + scenario.getName()+".log";
+        String scenarioLog = loggerDir.getPath() + File.separator + scenario.getScenarioName()+".log";
         scenarioLog = scenarioLog.replaceAll("\\s","_");
         FileAppender<ILoggingEvent> fileAppender = new FileAppender<ILoggingEvent>();
         fileAppender.setFile(scenarioLog);

engine-core/src/main/java/io/nosqlbench/engine/core/ScenarioResult.java

@@ -33,7 +33,6 @@ public class ScenarioResult {
 
     private Exception exception;
     private String iolog;
-    private String report;
 
     public ScenarioResult(String iolog) {
         this.iolog = iolog;

engine-core/src/main/java/io/nosqlbench/engine/core/ScenariosResults.java

@@ -29,8 +29,8 @@ public class ScenariosResults {
 
     private static final Logger logger = LoggerFactory.getLogger(ScenariosResults.class);
 
-    private String scenariosExecutorName;
-    private Map<Scenario, ScenarioResult> scenarioResultMap = new LinkedHashMap<>();
+    private final String scenariosExecutorName;
+    private final Map<Scenario, ScenarioResult> scenarioResultMap = new LinkedHashMap<>();
 
 
     public ScenariosResults(ScenariosExecutor scenariosExecutor) {
@@ -68,7 +68,7 @@ public class ScenariosResults {
             if (oresult != null) {
                 oresult.reportToLog();
             } else {
-                logger.error(scenario.getName() + ": incomplete (missing result)");
+                logger.error(scenario.getScenarioName() + ": incomplete (missing result)");
             }
         }
     }

engine-core/src/main/java/io/nosqlbench/engine/core/script/ScenariosExecutor.java

@@ -40,7 +40,7 @@ public class ScenariosExecutor {
     }
 
     public ScenariosExecutor(String name, int threads) {
-        executor = new ThreadPoolExecutor(1, 1,
+        executor = new ThreadPoolExecutor(1, threads,
                 0L, TimeUnit.MILLISECONDS,
                 new LinkedBlockingQueue<>(),
                 new IndexedThreadFactory("scenarios", new ScenarioExceptionHandler(this)));
@@ -53,8 +53,8 @@ public class ScenariosExecutor {
 
     public synchronized void execute(Scenario scenario, ScenarioLogger scenarioLogger) {
         scenario.setScenarioLogger(scenarioLogger);
-        if (submitted.get(scenario.getName()) != null) {
-            throw new BasicError("Scenario " + scenario.getName() + " is already defined. Remove it first to reuse the name.");
+        if (submitted.get(scenario.getScenarioName()) != null) {
+            throw new BasicError("Scenario " + scenario.getScenarioName() + " is already defined. Remove it first to reuse the name.");
         }
         Future<ScenarioResult> future = executor.submit(scenario);
         SubmittedScenario s = new SubmittedScenario(scenario, future);
@@ -229,7 +229,7 @@ public class ScenariosExecutor {
         }
 
         public String getName() {
-            return scenario.getName();
+            return scenario.getScenarioName();
         }
     }
 

engine-rest/pom.xml

@@ -0,0 +1,49 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>mvn-defaults</artifactId>
+        <groupId>io.nosqlbench</groupId>
+        <version>3.12.134-SNAPSHOT</version>
+        <relativePath>../mvn-defaults</relativePath>
+    </parent>
+
+  <artifactId>engine-rest</artifactId>
+  <packaging>jar</packaging>
+  <name>${project.artifactId}</name>
+  <description>REST services for nosqlbench</description>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <javadoc.name>nosqlbench REST Services</javadoc.name>
+  </properties>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>io.nosqlbench</groupId>
+      <artifactId>engine-cli</artifactId>
+      <version>3.12.134-SNAPSHOT</version>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.2.3</version>
+    </dependency>
+
+  </dependencies>
+
+</project>

engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorService.java

@@ -0,0 +1,210 @@
+package io.nosqlbench.engine.rest.resources;
+
+import io.nosqlbench.docsys.api.WebServiceObject;
+import io.nosqlbench.engine.cli.BasicScriptBuffer;
+import io.nosqlbench.engine.cli.Cmd;
+import io.nosqlbench.engine.cli.NBCLICommandParser;
+import io.nosqlbench.engine.cli.ScriptBuffer;
+import io.nosqlbench.engine.core.ScenarioResult;
+import io.nosqlbench.engine.core.script.Scenario;
+import io.nosqlbench.engine.core.script.ScenariosExecutor;
+import io.nosqlbench.engine.rest.transfertypes.RunScenarioRequest;
+import io.nosqlbench.engine.rest.transfertypes.ScenarioInfo;
+import io.nosqlbench.engine.rest.transfertypes.ResultInfo;
+import io.nosqlbench.nb.annotations.Service;
+
+import javax.inject.Singleton;
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.*;
+
+@Service(WebServiceObject.class)
+@Singleton
+@Path("/services/executor/")
+public class ScenarioExecutorService implements WebServiceObject {
+
+    private ScenariosExecutor executor = new ScenariosExecutor("executor-service", 1);
+
+    @POST
+    @Path("cli")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public synchronized Response invokeCommand(RunScenarioRequest rq) {
+
+        String name = rq.getScenarioName();
+        if (name.equals("auto")) {
+            rq.setScenarioName("scenario"+String.valueOf(System.currentTimeMillis()));
+        }
+
+        // First, virtualize files provided
+        storeFiles(rq);
+
+        LinkedList<Cmd> cmdList = new LinkedList<>();
+        LinkedList<String> args = new LinkedList<>(rq.getCommands());
+
+        for (String arg : args) {
+            if (arg.startsWith("-")) {
+                throw new RuntimeException("Only commands (verbs and params) can be used here");
+            }
+        }
+
+        args = substituteFilenames(rq, args);
+        NBCLICommandParser.parse(args, cmdList);
+        ScriptBuffer buffer = new BasicScriptBuffer();
+        buffer.add(cmdList.toArray(new Cmd[0]));
+
+        Scenario scenario = new Scenario(
+                rq.getScenarioName(),
+                "",
+                Scenario.Engine.Graalvm,
+                "disabled",
+                false,
+                true,
+                false
+        );
+        scenario.addScriptText(buffer.getParsedScript());
+
+        executor.execute(scenario);
+
+        return Response.created(UriBuilder.fromResource(ScenarioExecutorService.class).path(
+                "scenario/" + rq.getScenarioName()).build()).build();
+
+    }
+
+    private LinkedList<String> substituteFilenames(RunScenarioRequest rq, LinkedList<String> args) {
+        LinkedList<String> newargs = new LinkedList<>();
+        for (String arg : args) {
+            for (String s : rq.getFilemap().keySet()) {
+                arg = arg.replaceAll(s,rq.getFilemap().get(s));
+                newargs.add(arg);
+            }
+        }
+        return newargs;
+    }
+
+    private void storeFiles(RunScenarioRequest rq) {
+        Map<String, String> filemap = rq.getFilemap();
+
+        for (String filename : filemap.keySet()) {
+            try {
+                Paths.get(rq.getBasedir(),rq.getScenarioName());
+
+                Files.createDirectories(
+                        Paths.get(rq.getBasedir(),rq.getScenarioName()),
+                        PosixFilePermissions.asFileAttribute(
+                                PosixFilePermissions.fromString("rwxr-x---")
+                        ));
+                java.nio.file.Path tmpyaml = Files.createTempFile(
+                        Paths.get("/tmp/nosqlbench"),
+                        rq.getScenarioName(),
+                        filename
+                );
+//            // TODO: Find a better way to do this, like scoping resources to executor
+//            tmpyaml.toFile().deleteOnExit();
+
+                Files.write(
+                        tmpyaml,
+                        filemap.get(filename).getBytes(StandardCharsets.UTF_8)
+                );
+                rq.getFilemap().put(filename, tmpyaml.toString());
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+//    /**
+//     * Run a single-activity scenario
+//     *
+//     * @param scenarioName
+//     *         The name to install in the executor
+//     * @param params
+//     *         The params for the activity
+//     *
+//     * @return
+//     */
+//    @POST
+//    @Path("scenario/{scenarioName}")
+//    @Consumes(MediaType.APPLICATION_JSON)
+//    @Produces(MediaType.APPLICATION_JSON)
+//    public synchronized Response invokeScenario(
+//            @PathParam("scenarioName") String scenarioName,
+//            Map<String, String> params) {
+//        Scenario scenario = null;
+//        Optional<Scenario> pendingScenario = executor.getPendingScenario(scenarioName);
+//        if (pendingScenario.isPresent()) {
+//            scenario = pendingScenario.orElseThrow();
+//        } else {
+//            scenario = new Scenario(scenarioName, Scenario.Engine.Graalvm);
+//        }
+//        if (params.containsKey("yamldoc")) {
+//            try {
+//                java.nio.file.Path tmpyaml = Files.createTempFile(Paths.get("/tmp"), scenarioName, ".yaml");
+//                // TODO: Find a better way to do this, like scoping resources to executor
+//                tmpyaml.toFile().deleteOnExit();
+//                Files.write(tmpyaml, params.get("yamldoc").getBytes(StandardCharsets.UTF_8));
+//                params.remove("yamldoc");
+//                params.put("yaml", tmpyaml.toString());
+//            } catch (IOException e) {
+//                throw new RuntimeException(e);
+//            }
+//        }
+//        scenario.getScenarioController().apply(params);
+//        URI scenarioUri = UriBuilder.fromResource(ScenarioExecutorService.class)
+//                .build(scenarioName);
+//        return Response.created(scenarioUri).build();
+//    }
+
+    @GET
+    @Path("scenario/{scenarioName}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public synchronized ScenarioInfo getScenario(@PathParam("scenarioName") String scenarioName) {
+        Optional<Scenario> pendingScenario = executor.getPendingScenario(scenarioName);
+        if (pendingScenario.isPresent()) {
+            return new ScenarioInfo(pendingScenario.get());
+        } else {
+            throw new RuntimeException("Scenario name '" + scenarioName + "' not found.");
+        }
+    }
+
+    @GET
+    @Path("scenarios")
+    @Produces(MediaType.APPLICATION_JSON)
+    public synchronized List<ScenarioInfo> getScenarios() {
+        List<ScenarioInfo> scenarioInfos = new ArrayList<>();
+        List<String> pendingScenarios = executor.getPendingScenarios();
+        for (String pendingName : pendingScenarios) {
+            Optional<Scenario> pendingScenario = executor.getPendingScenario(pendingName);
+            pendingScenario.ifPresent(scenario -> scenarioInfos.add(new ScenarioInfo(scenario)));
+        }
+        return scenarioInfos;
+    }
+
+    @GET
+    @Path("result/{scenarioName}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public synchronized ResultInfo getResult(@PathParam("scenarioName") String scenarioName) {
+        return new ResultInfo(scenarioName, executor.getPendingResult(scenarioName).orElse(null));
+    }
+
+    @GET
+    @Path("results")
+    @Produces(MediaType.APPLICATION_JSON)
+    public synchronized List<ResultInfo> getResults() {
+        List<ResultInfo> results = new ArrayList<>();
+        List<String> pendingScenarios = executor.getPendingScenarios();
+        for (String pendingScenario : pendingScenarios) {
+            Optional<ScenarioResult> pendingResult = executor.getPendingResult(pendingScenario);
+            results.add(new ResultInfo(pendingScenario, pendingResult.orElse(null)));
+        }
+        return results;
+    }
+
+}

engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioTemplateService.java

@@ -1,4 +1,4 @@
-package io.nosqlbench.engine.services;
+package io.nosqlbench.engine.rest.resources;
 
 import io.nosqlbench.docsys.api.WebServiceObject;
 import io.nosqlbench.nb.annotations.Service;

engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultInfo.java

@@ -0,0 +1,30 @@
+package io.nosqlbench.engine.rest.transfertypes;
+
+import io.nosqlbench.engine.core.ScenarioResult;
+
+public class ResultInfo {
+    private final String scenarioName;
+    private final ScenarioResult result;
+
+    public ResultInfo(String scenarioName, ScenarioResult result) {
+        this.scenarioName = scenarioName;
+        this.result = result;
+    }
+
+    public String getScenarioName() {
+        return scenarioName;
+    }
+
+    public boolean isComplete() {
+        return result != null;
+    }
+
+    public boolean isErrored() {
+        return (result != null && result.getException().isPresent());
+    }
+
+    public String getIOLog() {
+        return result.getIOLog();
+    }
+
+}

engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/RunScenarioRequest.java

@@ -0,0 +1,70 @@
+package io.nosqlbench.engine.rest.transfertypes;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A CliRequest is what the user sends when they want to invoke NoSQLBench via web request in the
+ * same way they may on the command line.
+ *
+ * <pre>{@code
+ *  {
+ *    "name" : "auto",
+ *    "basedir" : "/tmp/nosqlbench",
+ *    "filemap" : {
+ *        "file1.yaml": "contents of file1"
+ *    },
+ *    "commands": [
+ *      "run", "workload=file1.yaml", "driver=stdout", "cycles=10M", "cyclerate=100"
+ *    ]
+ *  }
+ * }</pre>
+ */
+public class RunScenarioRequest {
+
+    private List<String> commands;
+    private Map<String, String> filemap;
+    private String stdout;
+    private String scenarioName = "auto";
+    private String basedir = "/tmp/nosqlbench";
+
+    public void setScenarioName(String scenarioName) {
+        this.scenarioName = scenarioName;
+    }
+
+    public String getScenarioName() {
+        return scenarioName;
+    }
+
+    public void setCommands(List<String> commands) {
+        this.commands = commands;
+    }
+
+    public void setFileMap(Map<String, String> filemap) {
+        this.filemap = filemap;
+    }
+
+    public void setStdout(String stdout) {
+        this.stdout = stdout;
+    }
+
+    public List<String> getCommands() {
+        return commands;
+    }
+
+    public Map<String, String> getFilemap() {
+        return filemap;
+    }
+
+    public String getStdout() {
+        return stdout;
+    }
+
+    public void setBasedir(String basedir) {
+        this.basedir = basedir;
+    }
+
+    public String getBasedir() {
+        return basedir;
+    }
+}

engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ScenarioInfo.java

@@ -0,0 +1,35 @@
+package io.nosqlbench.engine.rest.transfertypes;
+
+import io.nosqlbench.engine.api.activityapi.core.ProgressMeter;
+import io.nosqlbench.engine.core.script.Scenario;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ScenarioInfo {
+    private final Scenario scenario;
+
+    public ScenarioInfo(Scenario scenario) {
+        this.scenario = scenario;
+    }
+
+    public String getScenarioName() {
+        return scenario.getScenarioName();
+    }
+
+    public Map<String,String> getProgress() {
+        Map<String,String> progress = new HashMap<>();
+
+        Collection<ProgressMeter> progressMeters =
+                scenario.getScenarioController().getProgressMeters();
+        for (ProgressMeter meter : progressMeters) {
+            String activityName = meter.getProgressName();
+            String activityProgress = meter.getProgressDetails();
+            if (activityName!=null && activityProgress!=null) {
+                progress.put(activityName, activityProgress);
+            }
+        }
+        return progress;
+    }
+}

nb/pom.xml

@@ -32,6 +32,11 @@
             <artifactId>driver-kafka</artifactId>
             <version>3.12.134-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>engine-rest</artifactId>
+            <version>3.12.134-SNAPSHOT</version>
+        </dependency>
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>engine-cli</artifactId>

pom.xml

@@ -26,16 +26,19 @@
         <module>nb-api</module>
         <module>nb-annotations</module>
 
-        <!-- ENGINE MODULES -->
+        <!-- engine modules -->
         <module>engine-api</module>
         <module>engine-core</module>
-        <module>engine-cli</module>
         <module>engine-extensions</module>
         <module>engine-docker</module>
         <module>engine-docs</module>
+        <module>engine-cli</module>
+        <module>engine-rest</module>
 
         <!-- a binary (appimage) build option for nb -->
         <module>nb</module>
+
+        <!-- driver modules -->
         <module>driver-diag</module>
         <module>driver-stdout</module>
         <module>driver-tcp</module>