IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

merge fixups

Browse Source
This commit is contained in:
Jonathan Shook 2023-07-25 12:06:25 -05:00
commit d1725da0cb
38 changed files with 279 additions and 4472 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/build.yml vendored
View File

@ -44,18 +44,18 @@ jobs:
# Note: the target/coverage-report is not getting generated, involves pom settings not working as expected.
# - name: Collecting reports
# run: tar -cvf codecov-report.tar target/coverage-report/**/*
# run: tar -zcvf codecov-report.tgz target/coverage-report/**/*
#
# - name: Uploading test coverage
# uses: actions/upload-artifact@v3
# with:
# node-version: '16'
# name: codecov-report
# path: codecov-report.tar
# path: codecov-report.tgz
- name: Collecting logfiles
if: success() || failure()
run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
run: tar -zcvf logfiles.tgz [a-zA-Z]**/logs/*
- name: Uploading log files
if: success() || failure()
@ -63,7 +63,7 @@ jobs:
with:
node-version: '16'
name: nb-logs
path: logfiles.tar
path: logfiles.tgz
- name: export docs
if: success()

16
adapter-cqld4/pom.xml
View File

@ -49,12 +49,16 @@
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>3.6.4</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>tinkergraph-gremlin</artifactId>
<version>3.6.4</version>
</dependency>
<dependency>
@ -69,22 +73,24 @@
<version>4.17.0</version>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-core</artifactId>
</dependency>
<dependency>
<groupId>org.snakeyaml</groupId>
<artifactId>snakeyaml-engine</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.10.1</version>
</dependency>
<dependency>
<groupId>com.esri.geometry</groupId>
<artifactId>esri-geometry-api</artifactId>
<version>2.2.4</version>
</dependency>
<dependency>

5
adapter-cqld4/src/main/resources/ssl.md
View File

@ -54,3 +54,8 @@ For `openssl` type, the following options are available:
Examples:
- `keyFilePath=file.key`
- **sslValidation** - optional boolean flag to enable/disable SSLValidation.
Examples:
- `sslValidation=true` (the default)
- `sslValidation=false`

2
adapter-dynamodb/pom.xml
View File

@ -43,7 +43,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
<version>1.12.481</version>
<version>1.12.500</version>
</dependency>
</dependencies>

2
adapter-kafka/pom.xml
View File

@ -34,7 +34,7 @@
</description>
<properties>
<kafka.version>3.4.0</kafka.version>
<kafka.version>3.5.0</kafka.version>
</properties>
<dependencies>

2
adapter-mongodb/pom.xml
View File

@ -42,7 +42,7 @@
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
<version>4.9.1</version>
<version>4.10.1</version>
</dependency>
</dependencies>

4
docsys/pom.xml
View File

@ -106,12 +106,12 @@
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>4.0.2</version>
<version>4.0.3</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>4.0.2</version>
<version>4.0.3</version>
</dependency>
<dependency>

6
engine-api/pom.xml
View File

@ -92,23 +92,17 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</dependency>
<!-- test scope only -->
</dependencies>
<profiles>
<profile>
<id>perftests</id>

72
engine-api/src/test/java/io/nosqlbench/engine/api/util/SSLKsFactoryTest.java
View File

@ -178,8 +178,8 @@ public class SSLKsFactoryTest {
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to init KeyManagerFactory. Please check.*");
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageMatching("Unable to init KeyManagerFactory. Please check.*");
}
@Test
@ -196,6 +196,62 @@ public class SSLKsFactoryTest {
.withMessageMatching("Unable to load the truststore: .*");
}
@Test
void testSSLValidationActive() {
{
final String[] params1 = {
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key",
"sslValidation=true"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params1));
NBConfiguration sslCfg1 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThat(SSLKsFactory.get().getContext(sslCfg1)).isNotNull();
}
{
final String[] params2 = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench",
"sslValidation=true"
};
ActivityDef activityDef2 = ActivityDef.parseActivityDef(String.join(";", params2));
NBConfiguration sslCfg2 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef2.getParams());
assertThat(SSLKsFactory.get().getContext(sslCfg2)).isNotNull();
}
}
@Test
void testSSLValidationNotActive() {
{
final String[] params1 = {
"ssl=openssl",
"certFilePath=src/test/resources/ssl/client_cert.pem",
"keyFilePath=src/test/resources/ssl/client.key",
"sslValidation=false"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params1));
NBConfiguration sslCfg1 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThat(SSLKsFactory.get().getContext(sslCfg1)).isNotNull();
}
{
final String[] params2 = {
"ssl=jdk",
"keystore=src/test/resources/ssl/client_diff_password.p12",
"kspass=nosqlbench_client",
"keyPassword=nosqlbench",
"sslValidation=false"
};
ActivityDef activityDef2 = ActivityDef.parseActivityDef(String.join(";", params2));
NBConfiguration sslCfg2 = SSLKsFactory.get().getConfigModel().extractConfig(activityDef2.getParams());
assertThat(SSLKsFactory.get().getContext(sslCfg2)).isNotNull();
}
}
@Test
public void testOpenSSLGetContextWithCaCertError() {
String[] params = {
@ -241,16 +297,16 @@ public class SSLKsFactoryTest {
@Test
public void testOpenSSLGetContextWithMissingCertError() {
String[] params = {
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt",
"keyFilePath=src/test/resources/ssl/client.key"
"ssl=openssl",
"caCertFilePath=src/test/resources/ssl/cacert.crt",
"keyFilePath=src/test/resources/ssl/client.key"
};
ActivityDef activityDef = ActivityDef.parseActivityDef(String.join(";", params));
NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(activityDef.getParams());
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load key from")
.withCauseInstanceOf(IllegalArgumentException.class);
.isThrownBy(() -> SSLKsFactory.get().getContext(sslCfg))
.withMessageContaining("Unable to load key from")
.withCauseInstanceOf(IllegalArgumentException.class);
}
}

9
engine-docker/pom.xml
View File

@ -35,12 +35,10 @@
</properties>
<dependencies>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-api</artifactId>
@ -48,14 +46,15 @@
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-core</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-transport-okhttp</artifactId>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java</artifactId>

59
engine-rest/pom.xml
View File

@ -1,59 +0,0 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>${revision}</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<artifactId>engine-rest</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>REST services for nosqlbench</description>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<javadoc.name>nosqlbench REST Services</javadoc.name>
</properties>
<dependencies>
<dependency>
<groupId>io.swagger.parser.v3</groupId>
<artifactId>swagger-parser</artifactId>
<version>2.1.15</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-models</artifactId>
<version>2.2.11</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-cli</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</project>

95
engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/OpenApiEndpoint.java
View File

@ -1,95 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.resources;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.engine.rest.services.openapi.OpenApiLoader;
import io.nosqlbench.nb.annotations.Service;
import io.swagger.parser.OpenAPIParser;
import io.swagger.util.Json;
import io.swagger.v3.parser.converter.SwaggerConverter;
import jakarta.inject.Singleton;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import java.util.Map;
@Service(value = WebServiceObject.class, selector = "openapi")
@Singleton
@Path("/openapi")
public class OpenApiEndpoint implements WebServiceObject {
private final OpenAPIParser parser = new OpenAPIParser();
private final SwaggerConverter converter = new SwaggerConverter();
private final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private final Json sjson = new Json();
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("generate")
public Response putWorkload(String input) {
try {
return Response
.ok("received " + input.length() + " length request. Phase 2 implemention on the way...")
.build();
} catch (Exception e) {
return Response
.serverError()
.entity(e.getMessage())
.build();
}
}
@GET
@Path("paths")
@Produces(MediaType.APPLICATION_JSON)
public Response listPaths(@QueryParam("filepath") String filepath) {
try {
Map<String, Object> map = OpenApiLoader.parseToMap(filepath);
return Response.ok(Json.pretty(map)).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
// private String toJson(String method, String path, Operation op) {
// try {
// LinkedHashMap<String,Object> map = new LinkedHashMap<>();
// map.put("id", method.toUpperCase() + " " + path);
// map.put("path", path);
// map.put("method", method);
// map.put("description", op.getDescription());
// map.put("summary",op.getSummary());
// map.put("body",op.getRequestBody());
// map.put("parameters",op.getParameters());
// map.put("operation_id",op.getOperationId());
// map.put("external_docs",op.getExternalDocs());
//// return gson.toJson(map);
//
// } catch (Exception e) {
// throw new RuntimeException(e);
// }
// }
}

285
engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java
View File

@ -1,285 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.resources;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.engine.cli.BasicScriptBuffer;
import io.nosqlbench.engine.cli.Cmd;
import io.nosqlbench.engine.cli.NBCLICommandParser;
import io.nosqlbench.engine.cli.ScriptBuffer;
import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor;
import io.nosqlbench.engine.rest.services.WorkSpace;
import io.nosqlbench.engine.rest.services.WorkspaceFinder;
import io.nosqlbench.engine.rest.transfertypes.LiveScenarioView;
import io.nosqlbench.engine.rest.transfertypes.RunScenarioRequest;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import jakarta.inject.Singleton;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import java.io.CharArrayWriter;
import java.io.PrintWriter;
import java.util.*;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Service(value = WebServiceObject.class, selector = "scenario-executor")
@Singleton
@Path("/services/executor/")
public class ScenarioExecutorEndpoint implements WebServiceObject {
private final static Logger logger = LogManager.getLogger(ScenarioExecutorEndpoint.class);
private final ScenariosExecutor executor = new ScenariosExecutor("executor-service", 1);
@Context
private Configuration config;
@DELETE
@Path("scenario/{scenario}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public synchronized Response cancelScenario(@PathParam("scenario") String scenario) {
try {
executor.deleteScenario(scenario);
return Response.ok("canceled '" + scenario + "' and removed it").build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
@POST
@Path("stop/{scenario}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public synchronized Response stopScenario(@PathParam("scenario") String scenario) {
try {
executor.stopScenario(scenario, false);
return Response.ok("stopped '" + scenario + "' without removing it").build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
/**
* Run a NoSQLBench command just as you would on the command line. Certain parameters are translated
* (virtualized) into the workspace view for you automatically. That is, any path which would otherwise
* be resolved on the local file system will now be resolved in that same way but with the designated workspace
* as the base directory. All filesystem interaction which would otherwise happen in the current working
* directory should also be done relative to the designated workspace.
* @param rq
* @return
*/
@POST
@Path("cli")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public synchronized Response invokeCommand(RunScenarioRequest rq) {
String name = rq.getScenarioName();
if (name.equals("auto")) {
rq.setScenarioName("scenario" + System.currentTimeMillis());
}
org.joda.time.format.DateTimeFormatter dtf = DateTimeFormat.forPattern("yyyyMMddHHmmssSSS");
name = name.replaceAll("EPOCHMS", String.valueOf(System.currentTimeMillis()));
name = name.replaceAll("DATESTAMP", dtf.print(new DateTime()));
name = name.replaceAll("[:/ ]", "");
rq.setScenarioName(name);
WorkSpace workspace = new WorkspaceFinder(config).getWorkspace(rq.getWorkspace());
// First, virtualize files provided
storeFiles(rq);
LinkedList<Cmd> cmdList = new LinkedList<>();
LinkedList<String> args = new LinkedList<>(rq.getCommands());
for (String arg : args) {
if (arg.startsWith("-")) {
throw new RuntimeException("Only commands (verbs and params) can be used here");
}
}
args = substituteFilenames(rq, args);
Optional<List<Cmd>> parsed = NBCLICommandParser.parse(args, workspace.asIncludes());
if (!parsed.isPresent()) {
return Response.serverError().entity("Unable to render command stream from provided command spec.").build();
}
ScriptBuffer buffer = new BasicScriptBuffer();
buffer.add(cmdList.toArray(new Cmd[0]));
Scenario scenario = new Scenario(
rq.getScenarioName(),
"",
Scenario.Engine.Graalvm,
"disabled",
true,
false,
"",
cmdList.toString(),
(java.nio.file.Path) config.getProperties().get("logpath"),
Maturity.Unspecified);
scenario.addScriptText(buffer.getParsedScript());
executor.execute(scenario);
return Response.created(UriBuilder.fromResource(ScenarioExecutorEndpoint.class).path(
"scenario/" + rq.getScenarioName()).build()).entity("started").build();
}
private LinkedList<String> substituteFilenames(RunScenarioRequest rq, LinkedList<String> args) {
LinkedList<String> newargs = new LinkedList<>();
for (String arg : args) {
for (String s : rq.getFilemap().keySet()) {
Pattern basename = Pattern.compile(s);
String fullyQualifiedName = rq.getFilemap().get(s);
Matcher basenameMatcher = basename.matcher(arg);
StringBuilder newarg = new StringBuilder();
while (basenameMatcher.find()) {
basenameMatcher.appendReplacement(newarg,fullyQualifiedName);
}
basenameMatcher.appendTail(newarg);
arg = newarg.toString();
}
newargs.add(arg);
}
return newargs;
}
private void storeFiles(RunScenarioRequest rq) {
Map<String, String> filemap = rq.getFilemap();
if (filemap == null) {
return;
}
WorkspaceFinder ws = new WorkspaceFinder(config);
WorkSpace workspace = ws.getWorkspace(rq.getWorkspace());
Map<String, String> replacements = new HashMap<>();
for (String filename : filemap.keySet()) {
java.nio.file.Path targetPath = workspace.storeFile(filename, filemap.get(filename), replacements);
}
rq.setFileMap(replacements);
}
// /**
// * Run a single-activity scenario
// *
// * @param scenarioName
// * The name to install in the executor
// * @param params
// * The params for the activity
// *
// * @return
// */
// @POST
// @Path("scenario/{scenarioName}")
// @Consumes(MediaType.APPLICATION_JSON)
// @Produces(MediaType.APPLICATION_JSON)
// public synchronized Response invokeScenario(
// @PathParam("scenarioName") String scenarioName,
// Map<String, String> params) {
// Scenario scenario = null;
// Optional<Scenario> pendingScenario = executor.getPendingScenario(scenarioName);
// if (pendingScenario.isPresent()) {
// scenario = pendingScenario.orElseThrow();
// } else {
// scenario = new Scenario(scenarioName, Scenario.Engine.Graalvm);
// }
// if (params.containsKey("yamldoc")) {
// try {
// java.nio.file.Path tmpyaml = Files.createTempFile(Paths.get("/tmp"), scenarioName, ".yaml");
// // TODO: Find a better way to do this, like scoping resources to executor
// tmpyaml.toFile().deleteOnExit();
// Files.write(tmpyaml, params.get("yamldoc").getBytes(StandardCharsets.UTF_8));
// params.remove("yamldoc");
// params.put("yaml", tmpyaml.toString());
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// }
// scenario.getScenarioController().apply(params);
// URI scenarioUri = UriBuilder.fromResource(ScenarioExecutorService.class)
// .build(scenarioName);
// return Response.created(scenarioUri).build();
// }
/**
* Return a view of a named scenario, just as with {@link #getScenarios()}}.
* If the named scenario is not present, an error will be returned instead.
* @param scenarioName
* @return
*/
@GET
@Path("scenario/{scenarioName}")
@Produces(MediaType.APPLICATION_JSON)
public synchronized LiveScenarioView getScenario(@PathParam("scenarioName") String scenarioName) {
Optional<Scenario> pendingScenario = executor.getPendingScenario(scenarioName);
if (pendingScenario.isPresent()) {
Optional<Future<ExecutionMetricsResult>> pendingResult = executor.getPendingResult(scenarioName);
Future<ExecutionMetricsResult> scenarioResultFuture = pendingResult.get();
return new LiveScenarioView(pendingScenario.get());
} else {
throw new RuntimeException("Scenario name '" + scenarioName + "' not found.");
}
}
/**
* @return a view of all the scenarios known to the scenarios executor, whether starting,
* running, errored or otherwise. If the scenario is completed, then the result,
* including the IO log will be provided, otherwise an exception to explain why it failed.
*/
@GET
@Path("scenarios")
@Produces(MediaType.APPLICATION_JSON)
public synchronized Response getScenarios() {
try {
List<LiveScenarioView> liveScenarioViews = new ArrayList<>();
List<String> pendingScenarios = executor.getPendingScenarios();
for (String pendingScenario : pendingScenarios) {
LiveScenarioView liveScenarioView = getScenario(pendingScenario);
liveScenarioViews.add(liveScenarioView);
}
return Response.ok(liveScenarioViews).build();
} catch (Exception e) {
CharArrayWriter caw = new CharArrayWriter();
PrintWriter pw = new PrintWriter(caw);
e.printStackTrace(pw);
String trace = caw.toString();
return Response.serverError().entity(trace).build();
}
}
}

83
engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ServiceStatusEndpoint.java
View File

@ -1,83 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.resources;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.AutoDocsWebService;
import jakarta.inject.Singleton;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Configuration;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
@Service(value = WebServiceObject.class, selector = "service-status")
@Singleton
@Path("/services/status")
public class ServiceStatusEndpoint implements WebServiceObject {
private final static Logger logger = LogManager.getLogger(AutoDocsWebService.class);
@Context
private Configuration config;
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response isEnabled(@QueryParam("enabled") String overideEnabled) {
boolean enabled = false;
try {
if (overideEnabled != null) {
enabled = Boolean.parseBoolean(overideEnabled);
}
StatusEncoding status = new StatusEncoding(true, Map.of(
"status", "ORIGIN/services/status"
));
return Response.ok(status).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
private final static class StatusEncoding {
@JsonProperty("enabled")
public boolean isEnabled() {
return enabled;
}
@JsonProperty("endpoints")
public Map<String, String> getEndpoints() {
return endpoints;
}
private final boolean enabled;
private final Map<String, String> endpoints;
public StatusEncoding(boolean enabled, Map<String, String> endpoints) {
this.enabled = enabled;
this.endpoints = endpoints;
}
}
}

110
engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkloadFinderEndpoint.java
View File

@ -1,110 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.resources;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
import io.nosqlbench.engine.rest.services.WorkspaceFinder;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.AutoDocsWebService;
import jakarta.inject.Singleton;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Configuration;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service(value= WebServiceObject.class, selector="workload-finder")
@Singleton
@Path("/services/workloads")
public class WorkloadFinderEndpoint implements WebServiceObject {
private final static Logger logger = LogManager.getLogger(AutoDocsWebService.class);
@Context
private Configuration config;
public List<WorkloadDesc> getWorkloads(String search) {
return getWorkloads(Set.of(search!=null ? search.split(",") : new String[0]));
}
public List<WorkloadDesc> getWorkloads(Set<String> search) {
List<WorkloadDesc> workloads = new ArrayList<>();
WorkspaceFinder ws = new WorkspaceFinder(config);
for (String include : search) {
if (include.equals("builtins")) {
List<WorkloadDesc> activities = NBCLIScenarioParser.getWorkloadsWithScenarioScripts(true, "activities");
for (WorkloadDesc desc : activities) {
workloads.add(desc);
}
} else {
List<WorkloadDesc> descInWorkspace = ws.getWorkspace(include).getWorkloadsWithScenarioScripts();
for (WorkloadDesc workload : descInWorkspace) {
workloads.add(workload);
}
}
}
return workloads;
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getWorkloadDescriptions(@QueryParam("searchin") String searchin) {
WorkspaceFinder ws = new WorkspaceFinder(config);
Set<String> searchIn = Set.of(searchin != null ? searchin.split(",") : new String[]{});
try {
List<WorkloadDesc> workloads = getWorkloads(searchIn);
return Response.ok(workloads).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("parameters")
public Map<String, String> getParametersByWorkload(
@QueryParam("workloadName") String workloadName,
@QueryParam("searchin") String searchin
) {
List<WorkloadDesc> workloads = getWorkloads(searchin);
Map<String, String> templates = null;
templates = workloads.stream()
.filter(workload -> workload.getWorkloadName().equals(workloadName))
.map(workload -> workload.getTemplates())
.collect(Collectors.toSet()).iterator().next();
return templates;
}
}

176
engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkspacesEndpoint.java
View File

@ -1,176 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.resources;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.engine.rest.services.WorkSpace;
import io.nosqlbench.engine.rest.services.WorkspaceFinder;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceView;
import io.nosqlbench.nb.annotations.Service;
import jakarta.inject.Singleton;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Collectors;
@Service(value= WebServiceObject.class, selector="workspaces")
@Path("/services/workspaces")
@Singleton
public class WorkspacesEndpoint implements WebServiceObject {
private final static Logger logger = LogManager.getLogger(WorkspacesEndpoint.class);
@Context
private Configuration config;
private final static java.nio.file.Path workspacesRoot = Paths.get("workspaces");
private WorkspaceFinder svc;
/**
* @return A list of workspaces as a
* {@link List} of {@link WorkspaceView}
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getWorkspacesInfo() {
List<WorkspaceView> wsviews = getSvc().getWorkspaceViews();
return Response.ok(wsviews).build();
}
@DELETE
@Path("/{workspace}")
@Produces(MediaType.APPLICATION_JSON)
public Response deleteWorkspace(@PathParam("workspace") String workspace,
@QueryParam("deleteCount") String deleteCount) {
try {
int dc = deleteCount != null ? Integer.valueOf(deleteCount) : 0;
getSvc().purgeWorkspace(workspace, dc);
return Response.ok("removed workspace " + workspace).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
@GET
@Path("/{workspaceName}/{filepath:.+}")
public Response listFilesInWorkspace(
@PathParam("workspaceName") String workspaceName,
@PathParam("filepath") String filepath
) {
try {
WorkSpace w = getSvc().getWorkspace(workspaceName);
List<WorkspaceItemView> listing = w.getWorkspaceListingView(filepath);
return Response.ok(listing).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
@POST
@Path("/{workspaceName}/upload/{filepath}")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public Response uploadFileIntoWorkspace(
) {
return Response.ok().build();
}
@PUT
@Path("/{workspaceName}/{filepath:.+}")
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.WILDCARD)
public Response doSomething(@Context HttpServletRequest request, byte[] input) {
logger.debug("Content-Type: {}", request.getContentType());
logger.debug("Preferred output: {}", request.getHeader(HttpHeaders.ACCEPT));
try {
String pathInfo = request.getPathInfo();
String[] parts = pathInfo.split("/");
String workspaceName = parts[parts.length - 2];
String filename = parts[parts.length - 1];
getSvc().putFile(workspaceName, filename, ByteBuffer.wrap(input));
return Response.ok().build();
} catch (Exception e) {
return Response.status(Response.Status.BAD_REQUEST).build();
}
}
@GET
@Path("/{workspace}")
@Produces(MediaType.APPLICATION_JSON)
public Response getWorkspaceInfo(
@PathParam("workspace") String workspace,
@QueryParam("ls") String ls,
@QueryParam("contains") String contains
) {
try {
WorkSpace ws = getSvc().getWorkspace(workspace);
WorkspaceView wsview = ws.getWorkspaceView();
if (ls != null && !ls.equalsIgnoreCase("false")) {
List<WorkspaceItemView> listing = ws.getWorkspaceListingView("");
if (contains != null) {
listing = listing.stream().filter(i -> i.contains(contains)).collect(Collectors.toList());
}
wsview.setListing(listing);
}
return Response.ok(wsview).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
@GET
@Path("/{workspace}/{filename}")
public Response getFileInWorkspace(
@PathParam("workspace") String workspace,
@PathParam("filename") String filename,
@QueryParam("ls") String ls) {
try {
if (ls != null && !ls.equalsIgnoreCase("false")) {
WorkSpace ws = getSvc().getWorkspace(workspace);
List<WorkspaceItemView> listing = ws.getWorkspaceListingView(filename);
return Response.ok(listing).build();
} else {
WorkspaceFinder.FileInfo fileinfo = getSvc().readFile(workspace, filename);
if (fileinfo != null) {
return Response.ok(fileinfo.getPath().toFile(), fileinfo.getMediaType()).build();
} else {
return Response.noContent().status(Response.Status.NOT_FOUND).build();
}
}
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
private WorkspaceFinder getSvc() {
if (svc == null) {
svc = new WorkspaceFinder(config);
}
return svc;
}
}

172
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkSpace.java
View File

@ -1,172 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.services;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceView;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
public class WorkSpace {
private final Path workspacesRoot;
private final Path workspacePath;
private final String workspaceName;
public WorkSpace(Path workspacesRoot, String workspaceName) {
this.workspacesRoot = workspacesRoot;
this.workspaceName = workspaceName;
this.workspacePath = workspacesRoot.resolve(workspaceName);
if (!Files.exists(workspacePath)) {
try {
Files.createDirectories(workspacePath);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public WorkspaceView getWorkspaceView() {
return new WorkspaceView(workspacesRoot.resolve(workspaceName));
}
public Path getWorkspacePath() {
return workspacePath;
}
public List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
List<Content<?>> candidates = NBIO.fs().prefix(this.getWorkspacePath().toString()).extension(RawStmtsLoader.YAML_EXTENSIONS).list();
List<WorkloadDesc> workloads = NBCLIScenarioParser.filterForScenarios(candidates);
List<WorkloadDesc> relativized = new ArrayList<>();
for (WorkloadDesc workload : workloads) {
WorkloadDesc relative = workload.relativize(getWorkspacePath());
relativized.add(relative);
}
return relativized;
}
public Path storeFile(String filespec, String encoded, Map<String,String> replacements) {
String encoding = "raw";
String filename = filespec;
String[] parts = filespec.split(":", 2);
if (parts.length == 2) {
filename = parts[0];
encoding = parts[1].toLowerCase();
}
List<OpenOption> openOptions = new ArrayList<>();
if (filename.startsWith(">>")) {
filename = filename.substring(">>".length());
openOptions.add(StandardOpenOption.CREATE);
openOptions.add(StandardOpenOption.APPEND);
} else if (filename.startsWith(">")) {
filename = filename.substring(">".length());
openOptions.add(StandardOpenOption.TRUNCATE_EXISTING);
openOptions.add(StandardOpenOption.CREATE);
}
Path targetPath = Paths.get(filename);
assertLegalWorkspacePath(targetPath);
if (targetPath.isAbsolute()) {
throw new RuntimeException("You may not use absolute paths in workspaces: '" + targetPath + "'");
}
targetPath = this.workspacePath.resolve(targetPath);
ByteBuffer content;
switch (encoding) {
case "raw":
content = ByteBuffer.wrap(encoded.getBytes(StandardCharsets.UTF_8));
break;
case "base64":
byte[] bytes = Base64.getDecoder().decode(encoded);
content = ByteBuffer.wrap(bytes);
break;
default:
throw new RuntimeException("Unrecognized encoding of file data '" + encoding + "'");
}
try {
Files.createDirectories(
targetPath.getParent(),
PosixFilePermissions.asFileAttribute(
PosixFilePermissions.fromString("rwxr-x---")
));
Files.write(targetPath, content.array(), openOptions.toArray(new OpenOption[0]));
replacements.put(filename,targetPath.toString());
return targetPath;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public List<WorkspaceItemView> getWorkspaceListingView(String filepath) {
Path target = this.workspacePath.resolve(filepath);
assertLegalWorkspacePath(target);
List<WorkspaceItemView> items = new ArrayList<>();
try (DirectoryStream<Path> elementPaths = Files.newDirectoryStream(target)) {
for (Path elementPath : elementPaths) {
items.add(new WorkspaceItemView(this.workspacePath,elementPath));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return items;
}
private void assertLegalWorkspacePath(Path target) {
if (!target.normalize().startsWith(this.workspacePath)) {
throw new RuntimeException("workspace path '" + target + "' contains path traversal");
}
if (target.toString().contains("..")) {
throw new RuntimeException("Possible path injection:" + target);
}
}
@Override
public String toString() {
return this.workspaceName;
}
public String[] asIncludes() {
Path relativePath =
this.workspacesRoot.toAbsolutePath().getParent().relativize(this.workspacePath.toAbsolutePath());
return new String[]{ relativePath.toString() };
}
}

206
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkspaceFinder.java
View File

@ -1,206 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.services;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceView;
import jakarta.ws.rs.core.Configuration;
import jakarta.ws.rs.core.MediaType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Stream;
public class WorkspaceFinder {
private final static Logger logger = LogManager.getLogger(WorkspaceFinder.class);
public static String DEFAULT = "default";
public static final String WORKSPACE_ROOT = "workspaces_root";
private final Path root;
public WorkspaceFinder(Configuration config) {
Object root = config.getProperties().get(WORKSPACE_ROOT);
if (root instanceof Path) {
this.root = (Path) root;
} else if (root instanceof CharSequence) {
this.root = Paths.get(((CharSequence) root).toString());
} else if (root == null) {
this.root = Paths.get(
System.getProperty("user.dir"),
"workspaces"
);
try {
Files.createDirectories(this.root);
} catch (IOException e) {
throw new RuntimeException(e);
}
} else {
throw new RuntimeException("Unable to use workspaces root " +
"path of type " + root.getClass().getCanonicalName());
}
createDefaultIfNotExist();
}
public WorkspaceFinder(Path root) {
this.root = root;
createDefaultIfNotExist();
}
private void createDefaultIfNotExist() {
getWorkspaceView(DEFAULT);
}
public List<WorkspaceView> getWorkspaceViews() {
List<WorkspaceView> views = new ArrayList<>();
try (DirectoryStream<Path> wsrEntries = Files.newDirectoryStream(root)) {
for (Path entry : wsrEntries) {
views.add(new WorkspaceView(entry));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return views;
}
public WorkSpace getWorkspace(String workspaceName) {
assertLegalWorkspaceName(workspaceName);
return new WorkSpace(this.root, workspaceName);
}
public static void assertLegalWorkspaceName(String workspaceName) {
if (!workspaceName.matches("[a-zA-Z0-9]+")) {
throw new RuntimeException("Workspace names must contain only letters and numbers.");
}
}
public WorkspaceView getWorkspaceView(String workspaceName) {
return getWorkspace(workspaceName).getWorkspaceView();
}
public void putFile(String workspaceName, String filename, ByteBuffer content) {
Path toWrite = root.resolve(workspaceName).resolve(filename);
try {
Files.write(toWrite, content.array(),
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Read the bytes of the named file in the named workspace.
*
* @param workspaceName The workspace name to look in for the file
* @param filename The filename within the workspace to read
* @return null if the file is not found
* @throws RuntimeException if the file was found but could not be
* read.
*/
public FileInfo readFile(String workspaceName, String filename) {
Path filePath = workspacePath(workspaceName).resolve(filename);
if (Files.exists(filePath)) {
return new FileInfo(filePath);
} else {
return null;
}
}
private Path workspacePath(String workspaceName) {
return root.resolve(workspaceName);
}
public void purgeWorkspace(String workspaceName, int deleteCount) {
assertLegalWorkspaceName(workspaceName);
Path path = workspacePath(workspaceName);
if (Files.exists(path)) {
try (Stream<Path> counter = Files.walk(path)) {
long foundFiles = counter.count();
if (foundFiles > 100 && deleteCount != foundFiles) {
throw new RuntimeException(
"To delete " + foundFiles + " files, you must provide a deleteCount=<count> " +
"parameter that matches. This is a safety mechanism."
);
}
logger.debug(() -> "found " + foundFiles + " to delete.");
} catch (Exception e) {
throw new RuntimeException(e);
}
Path relativize = root.relativize(path);
if (relativize.toString().contains("..")) {
throw new RuntimeException("Illegal path to delete: " + path);
}
try (Stream<Path> walk = Files.walk(path)) {
walk.sorted(Comparator.reverseOrder())
.map(Path::toFile)
// .peek(System.out::println)
.forEach(f -> {
logger.debug(() -> "deleting '" + f + "'");
if (!f.delete()) {
throw new RuntimeException("Unable to delete " + f);
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public final static class FileInfo {
private final Path path;
public FileInfo(Path path) {
this.path = path;
}
public MediaType getMediaType() {
try {
String contentType = Files.probeContentType(path);
MediaType mediaType = MediaType.valueOf(contentType);
return mediaType;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public ByteBuffer getContent() {
byte[] bytes = new byte[0];
try {
bytes = Files.readAllBytes(path);
} catch (IOException e) {
throw new RuntimeException(e);
}
return ByteBuffer.wrap(bytes);
}
public Path getPath() {
return path;
}
}
}

178
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/openapi/OpenApiLoader.java
View File

@ -1,178 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.services.openapi;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.swagger.parser.OpenAPIParser;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.Paths;
import io.swagger.v3.parser.core.models.ParseOptions;
import io.swagger.v3.parser.core.models.SwaggerParseResult;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.introspector.BeanAccess;
import java.util.*;
import java.util.stream.Collectors;
/**
* @see <A href="https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.1.0
* .md#securityRequirementObject">OpenApi Spec 3.1.0</A>
*/
public class OpenApiLoader {
private final static Logger logger = LogManager.getLogger(OpenApiLoader.class);
private static final OpenAPIParser parser = new OpenAPIParser();
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
/**
* If it is not found, the Stargate path is used by default.
*
* For now, the json is used as a filter with the following conventions:
* <pre>{@code
* {
* "<method> <path>": {...},
* ...
* }
* }</pre>
*
* The presence of a key with a matching method and path to one of those found
* in the openapi details indicates it should be included.
*
* @param openIdYamlPath a filepath is where an openapi descriptor can be found
* @param json The selection data used to filer in our out calls from the openapi descriptor
* @return A yaml workload which can be used with the http driver
*/
public static String generateWorkloadFromFilepath(String openIdYamlPath, String json) {
Map<?, ?> filter = gson.fromJson(json, Map.class);
Set<String> included = filter.keySet().stream()
.map(Object::toString)
.collect(Collectors.toSet());
OpenAPI openAPI = parseOpenApi(openIdYamlPath);
Paths paths = openAPI.getPaths();
List<PathOp> allOps = new ArrayList<>();
for (String pathName : paths.keySet()) {
PathItem pathItem = paths.get(pathName);
List<PathOp> calls = PathOp.wrap(pathName, pathItem);
allOps.addAll(calls);
}
List<PathOp> activeOps = allOps.stream()
.filter(op -> {
return included.contains(op.getCall());
})
.collect(Collectors.toList());
Map<String, PathOp> pathops = new HashMap<>();
DumperOptions dumper = new DumperOptions();
dumper.setAllowReadOnlyProperties(true);
Yaml yaml = new Yaml(dumper);
yaml.setBeanAccess(BeanAccess.DEFAULT);
for (PathOp activeOp : activeOps) {
logger.debug(() -> "yaml for op:" + yaml.dump(activeOp));
pathops.put(activeOp.getCall(), activeOp);
}
String dump = yaml.dump(pathops);
return dump;
}
public static OpenAPI parseOpenApi(String filepath) {
if (filepath == null) {
filepath = "stargate.yaml";
} else if (!filepath.endsWith(".yaml")) {
throw new RuntimeException("Only .yaml filepaths are supported for now.");
}
Content<?> one = NBIO.all().name(filepath).one();
String content = one.asString();
SwaggerParseResult parsed = parser.readContents(content, List.of(), new ParseOptions());
List<String> messages = parsed.getMessages();
if (messages.size() > 0) {
throw new RuntimeException("error while parsing: " + String.join("\n", messages.toArray(new String[0])));
}
OpenAPI openAPI = parsed.getOpenAPI();
return new OpenApiView(openAPI).getDereferenced();
}
// TODO; use the op wrapper interface here
public static Map<String, Object> parseToMap(String filepath) {
OpenAPI openAPI = parseOpenApi(filepath);
Paths paths = openAPI.getPaths();
LinkedHashMap<String, Object> map = new LinkedHashMap<>();
for (String pathName : paths.keySet()) {
PathItem pathItem = paths.get(pathName);
if (pathItem.getGet() != null) {
Operation op = pathItem.getGet();
map.put("GET " + pathName, map("GET", pathName, op));
}
if (pathItem.getPost() != null) {
Operation op = pathItem.getPost();
map.put("POST " + pathName, map("POST", pathName, op));
}
if (pathItem.getPut() != null) {
Operation op = pathItem.getPut();
map.put("PUT " + pathName, map("PUT", pathName, op));
}
if (pathItem.getPatch() != null) {
Operation op = pathItem.getPatch();
map.put("PATCH " + pathName, map("PATCH", pathName, op));
}
if (pathItem.getDelete() != null) {
Operation op = pathItem.getDelete();
map.put("DELETE " + pathName, map("DELETE", pathName, op));
}
if (pathItem.getHead() != null) {
Operation op = pathItem.getHead();
map.put("HEAD " + pathName, map("HEAD", pathName, op));
}
if (pathItem.getOptions() != null) {
Operation op = pathItem.getOptions();
map.put("OPTIONS " + pathName, map("OPTIONS", pathName, op));
}
}
return map;
}
private static Map<String, Object> map(String method, String pathName, Operation op) {
LinkedHashMap<String, Object> map = new LinkedHashMap<>();
map.put("method", method);
map.put("path", pathName);
map.put("api", op);
map.put("summary", op.getSummary() != null ? op.getSummary() : "");
map.put("description", op.getDescription() != null ? op.getDescription() : "");
return map;
}
}

280
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/openapi/OpenApiView.java
View File

@ -1,280 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.services.openapi;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.Paths;
import io.swagger.v3.oas.models.headers.Header;
import io.swagger.v3.oas.models.links.Link;
import io.swagger.v3.oas.models.media.Schema;
import io.swagger.v3.oas.models.parameters.Parameter;
import io.swagger.v3.oas.models.parameters.RequestBody;
import io.swagger.v3.oas.models.responses.ApiResponse;
import io.swagger.v3.oas.models.responses.ApiResponses;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#pathsObject
*/
public class OpenApiView {
private final OpenAPI model;
public OpenApiView(OpenAPI model) {
this.model = model;
}
public OpenAPI getDereferenced() {
// Since links, parameters, and responses in components can have references,
// connect their references first
resolveBodiesInLinks();
resolveSchemasInParameters();
resolveHeadersInResponses();
// OpenAPI out = new OpenAPI();
Paths paths = model.getPaths();
for (String pathKey : paths.keySet()) {
PathItem pi = paths.get(pathKey);
if (pi.get$ref() != null) {
throw new RuntimeException("Unable to read external ref in this version for path '" + pathKey + "':'"
+ pi.get$ref() + "'");
}
for (Operation op : new Operation[]{
pi.getDelete(),
pi.getGet(),
pi.getPost(),
pi.getTrace(),
pi.getPatch(),
pi.getOptions(),
pi.getPut(),
pi.getHead()
}) {
if (op != null) {
flattenOperation(op);
}
}
pi.setParameters(resolveParameterList(pi.getParameters()));
}
return model;
}
private void flattenOperation(Operation op) {
if (op.getResponses() != null) {
op.setResponses(resolveResponsesMap(op.getResponses()));
}
if (op.getParameters() != null) {
op.setParameters(resolveParameterList(op.getParameters()));
}
if (op.getRequestBody() != null) {
op.setRequestBody(resolveRequestBody(op.getRequestBody()));
}
}
private RequestBody resolveRequestBody(RequestBody requestBody) {
while (requestBody.get$ref() != null) {
requestBody = model.getComponents().getRequestBodies().get(requestBody.get$ref());
}
return requestBody;
}
private List<Parameter> resolveParameterList(List<Parameter> parameters) {
if (parameters == null) {
return null;
}
List<Parameter> resolved = new ArrayList<>();
for (Parameter p : parameters) {
p = resolve(p);
p.setSchema(resolveSchema(p.getSchema()));
resolved.add(p);
}
return resolved;
}
private final static Map<Class<?>, String> componentPaths =
Map.of(
Parameter.class, "#/components/parameters/",
ApiResponse.class, "#/components/responses/"
);
private final static Map<Class<?>, String> mapMethods =
Map.of(
Parameter.class, "getParameters",
ApiResponse.class, "getResponses"
);
private <T> T resolve(T aliased) {
if (aliased == null) {
return null;
}
String typepath = componentPaths.get(aliased.getClass());
if (typepath == null) {
throw new RuntimeException("Could not find component path prefix for " + aliased.getClass().getCanonicalName());
}
String mapMethod = mapMethods.get(aliased.getClass());
if (mapMethod == null) {
throw new RuntimeException("Could not find map method for " + aliased.getClass().getCanonicalName());
}
T element = aliased;
int remaining = 100;
while (true) {
if (remaining <= 0) {
throw new RuntimeException("loop limit reached in resolving element");
}
try {
Method getref = element.getClass().getMethod("get$ref");
Object invoke = getref.invoke(element);
if (invoke == null) {
return element;
}
String refid = invoke.toString();
int idAt = refid.lastIndexOf("/");
String name = refid.substring(idAt + 1);
String prefix = refid.substring(0, idAt + 1);
if (!prefix.equals(typepath)) {
throw new RuntimeException("wrong type path (" + typepath + ") for prefix '" + prefix + "'");
}
Method getMap = model.getComponents().getClass().getMethod(mapMethod);
Object mapobj = getMap.invoke(model.getComponents());
Map map = (Map) mapobj;
Object o = map.get(name);
element = (T) o;
} catch (Exception e) {
throw new RuntimeException("unable to call get$ref: " + aliased.getClass().getCanonicalName());
}
}
}
private Schema resolveSchema(Schema schema) {
while (schema.get$ref() != null) {
schema = model.getComponents().getSchemas().get(schema.get$ref());
}
return schema;
}
private ApiResponses resolveResponsesMap(ApiResponses responses) {
if (responses != null) {
for (String rk : responses.keySet()) {
ApiResponse response = responses.get(rk);
response = resolve(response);
response.setHeaders(resolveHeaderMap(response.getHeaders()));
response.setExtensions(resolveExtensionsMap(response.getExtensions()));
response.setLinks(resolveLinksMap(response.getLinks()));
}
}
return responses;
}
private Map<String, Link> resolveLinksMap(Map<String, Link> links) {
if (links != null) {
for (String lk : links.keySet()) {
Link link = links.get(lk);
while (link.get$ref() != null) {
link = model.getComponents().getLinks().get(link.get$ref());
}
links.put(lk, link);
}
}
return links;
}
private Map<String, Object> resolveExtensionsMap(Map<String, Object> extensions) {
if (extensions != null) {
if (extensions.keySet().size() > 0) {
throw new RuntimeException("extensions are not supported in this version");
}
}
return extensions;
}
private Map<String, Header> resolveHeaderMap(Map<String, Header> headers) {
if (headers != null) {
for (String hk : headers.keySet()) {
Header header = headers.get(hk);
while (header.get$ref() != null) {
header = model.getComponents().getHeaders().get(hk);
}
headers.put(hk, header);
}
}
return headers;
}
private void resolveBodiesInLinks() {
Map<String, Link> links = model.getComponents().getLinks();
if (links == null) {
return;
}
for (String linkKey : links.keySet()) {
Link modelLink = model.getComponents().getLinks().get(linkKey);
// RequestBody body = modelLink.getRequestBody();
// while (body.get$ref() != null) {
// body = model.getComponents().getRequestBodies().get(body.get$ref());
// }
Object body = modelLink.getRequestBody();
modelLink.setRequestBody(body);
}
}
private void resolveSchemasInParameters() {
for (String parameterKey : model.getComponents().getParameters().keySet()) {
Parameter parameter = model.getComponents().getParameters().get(parameterKey);
Schema schema = parameter.getSchema();
while (schema.get$ref() != null) {
schema = model.getComponents().getSchemas().get(schema.get$ref());
}
parameter.setSchema(schema);
}
}
private void resolveHeadersInResponses() {
for (String responseKey : model.getComponents().getResponses().keySet()) {
ApiResponse response = model.getComponents().getResponses().get(responseKey);
Map<String, Header> modelHeaders = response.getHeaders();
Map<String, Header> headers = new HashMap<>();
for (String headerKey : headers.keySet()) {
Header header = modelHeaders.get(headerKey);
while (header.get$ref() != null) {
header = modelHeaders.get(header.get$ref());
}
headers.put(headerKey, header);
}
response.setHeaders(headers);
}
}
}

91
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/openapi/PathOp.java
View File

@ -1,91 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.services.openapi;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.parameters.Parameter;
import java.util.ArrayList;
import java.util.List;
public class PathOp {
private final String method;
private final String path;
private final Operation op;
public PathOp(String method, String path, Operation op) {
this.method = method;
this.path = path;
this.op = op;
}
public Operation getOp() {
return op;
}
public static List<PathOp> wrap(String path, PathItem item) {
List<String> methods = List.of("GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD", "TRACE");
Operation[] ops = new Operation[]{item.getGet(), item.getPost(), item.getPut(), item.getDelete(),
item.getPatch(), item.getOptions(),
item.getHead(), item.getTrace()};
List<PathOp> pathops = new ArrayList<>();
for (int i = 0; i < methods.size(); i++) {
PathOp pathop = wrap(path, methods.get(i), ops[i]);
if (pathop != null) {
pathops.add(pathop);
}
}
return pathops;
}
private static PathOp wrap(String path, String method, Operation op) {
if (op == null) {
return null;
}
return new PathOp(method, path, op);
}
public Operation getOperation() {
return this.op;
}
public String toString() {
String call = getMethod() + " " + getPath();
if (getOperation().getParameters().size() > 0) {
for (Parameter p : getOperation().getParameters()) {
String name = p.getName();
System.out.println("name: " + name);
}
}
return call;
}
public String getPath() {
return this.path;
}
public String getMethod() {
return this.method;
}
public String getCall() {
return method.toUpperCase() + " " + path;
}
}

27
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/package-info.java
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Classes in this package are meant to provide a basic
* internal service facade to be used by endpoints.
* This simplifies endpoint implementations and facilitates DRY
* implementations.
*
* These implementations should only expose primitive types,
* collections of primitive types, or views of transfer types
* as implemented in that package.
*/
package io.nosqlbench.engine.rest.services;

78
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/LiveScenarioView.java
View File

@ -1,78 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
import io.nosqlbench.engine.core.lifecycle.scenario.Scenario;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
public class LiveScenarioView {
private final Scenario scenario;
public LiveScenarioView(Scenario scenario) {
this.scenario = scenario;
}
@JsonProperty
@JsonPropertyDescription("Optionally populated result, "+
" present only if there was an error or the scenario is complete")
public ResultView getResult() {
return new ResultView(scenario.getResultIfComplete().orElse(null));
}
@JsonProperty("scenario_name")
public String getScenarioName() {
return scenario.getScenarioName();
}
@JsonProperty("started_at")
public long getStartMillis() {
return scenario.getStartedAtMillis();
}
@JsonProperty("ended_at")
public long getEndMillis() {
return scenario.getEndedAtMillis();
}
public Scenario.State getState() {
return scenario.getScenarioState();
}
@JsonProperty("progress")
public List<ProgressView> getProgress() {
List<ProgressView> progressView = new ArrayList<>();
if (scenario.getScenarioController()==null) {
return progressView;
}
Collection<? extends ProgressMeterDisplay> meters = scenario.getScenarioController().getProgressMeters();
for (ProgressMeterDisplay progressMeterDisplay : meters) {
ProgressView meterView = new ProgressView(progressMeterDisplay);
progressView.add(meterView);
}
return progressView;
}
}

99
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ProgressView.java
View File

@ -1,99 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.nosqlbench.engine.api.activityapi.core.progress.CycleMeter;
import io.nosqlbench.engine.api.activityapi.core.progress.ProgressMeterDisplay;
import io.nosqlbench.engine.api.activityapi.core.progress.StateCapable;
public class ProgressView {
private final ProgressMeterDisplay progressMeterDisplay;
public ProgressView(ProgressMeterDisplay progressMeterDisplay) {
if (progressMeterDisplay ==null) {
throw new RuntimeException("Unable to create a view with a null progressMeter");
}
this.progressMeterDisplay = progressMeterDisplay;
}
@JsonProperty("summary")
public String getProgressDetails() {
return progressMeterDisplay.getSummary();
}
@JsonProperty("min")
public double getMin() {
return progressMeterDisplay.getMinValue();
}
@JsonProperty("current")
public double getCurrent() {
return progressMeterDisplay.getCurrentValue();
}
@JsonProperty("max")
public double getMax() {
return progressMeterDisplay.getMaxValue();
}
@JsonProperty("recycles_max")
public double getRecyclesMax() {
if (progressMeterDisplay instanceof CycleMeter cm) {
return cm.getRecyclesMax();
} else {
return Double.NaN;
}
}
@JsonProperty("recycles_current")
public double getRecyclesCurrent() {
if (progressMeterDisplay instanceof CycleMeter cm) {
return cm.getRecyclesCurrent();
} else {
return Double.NaN;
}
}
@JsonProperty("eta_millis")
public double getEtaMills() {
return progressMeterDisplay.getProgressETAMillis();
}
@JsonProperty("name")
public String getName() {
return progressMeterDisplay.getProgressName();
}
@JsonProperty("completed")
public double getProgress() {
return progressMeterDisplay.getRatioComplete();
}
@JsonProperty("state")
public String getState() {
if (progressMeterDisplay instanceof StateCapable) {
return ((StateCapable) progressMeterDisplay).getRunState().toString();
} else {
return "unknown";
}
}
}

43
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/ResultView.java
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import io.nosqlbench.engine.core.lifecycle.ExecutionMetricsResult;
public class ResultView {
private final ExecutionMetricsResult result;
public ResultView(ExecutionMetricsResult result) {
this.result = result;
}
public String getIOLog() {
if (result != null) {
return result.getIOLog();
} else {
return "";
}
}
public String getError() {
if (result != null && result.getException()!=null) {
return result.getException().getMessage();
}
return "";
}
}

81
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/RunScenarioRequest.java
View File

@ -1,81 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
import java.util.Map;
public class RunScenarioRequest {
@JsonProperty("commands")
private List<String> commands = List.of();
@JsonProperty("filemap")
private Map<String, String> filemap = Map.of();
@JsonProperty("console")
private String stdout;
@JsonProperty("scenario_name")
private String scenarioName = "auto";
@JsonProperty("workspace")
private String workspace = "default";
public String getWorkspace() {
return workspace;
}
public void setWorkspace(String workspace) {
this.workspace = workspace;
}
public void setScenarioName(String scenarioName) {
this.scenarioName = scenarioName;
}
public String getScenarioName() {
return scenarioName;
}
public void setCommands(List<String> commands) {
this.commands = commands;
}
public void setFileMap(Map<String, String> filemap) {
this.filemap = filemap;
}
public void setStdout(String stdout) {
this.stdout = stdout;
}
public List<String> getCommands() {
return commands;
}
public Map<String, String> getFilemap() {
return filemap;
}
public String getStdout() {
return stdout;
}
}

42
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkloadsView.java
View File

@ -1,42 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class WorkloadsView {
private final Map<String, List<WorkloadDesc>> workloadsByWorkspace = new HashMap<>();
@JsonProperty("workloads")
public Map<String,List<WorkloadDesc>> getWorkloads() {
return workloadsByWorkspace;
}
public void add(String workspace, WorkloadDesc workload) {
workloadsByWorkspace.computeIfAbsent(workspace, ws -> new ArrayList<>()).add(workload);
}
public void addAll(String workspace, List<WorkloadDesc> workloads) {
workloadsByWorkspace.computeIfAbsent(workspace,ws -> new ArrayList<>()).addAll(workloads);
}
}

188
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspaceItemView.java
View File

@ -1,188 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileOwnerAttributeView;
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
public class WorkspaceItemView {
private final Path _path;
private String type;
private String perms;
private String owner;
private String group;
private long size;
private long mtime;
private final static List<String> fields = List.of(
"type",
"perms",
"owner",
"group",
"size",
"mtime",
"name"
);
public WorkspaceItemView(Path wspath, Path path) {
try {
PosixFileAttributeView posix = Files.getFileAttributeView(path, PosixFileAttributeView.class);
PosixFileAttributes attrs = posix.readAttributes();
setPerms(fromPerms(attrs.permissions()));
setType(typeOf(path));
setOwner(attrs.owner().getName());
setGroup(attrs.group().getName());
setSize(attrs.size());
setMtimeMillis(attrs.lastModifiedTime().to(TimeUnit.MILLISECONDS));
setName(wspath.relativize(path).toString());
this._path = path;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private void setOwner(FileOwnerAttributeView fileAttributeView) {
try {
this.setOwner(fileAttributeView.getOwner().getName());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private String typeOf(Path path) {
if (Files.isRegularFile(path)) {
return "F";
}
if (Files.isDirectory(path)) {
return "D";
}
return "U";
}
@JsonProperty("fields")
public List<List<String>> getAsFields() {
return List.of(
fields,
List.of(
this.type,
this.perms,
this.owner,
this.group,
String.valueOf(this.size),
"mtime",
this.name
)
);
}
private String fromPerms(Set<PosixFilePermission> perms) {
StringBuilder sb = new StringBuilder();
String img = "rwxrwxrwx";
String not = "---------";
int step = 0;
for (PosixFilePermission perm : PosixFilePermission.values()) {
String src = perms.contains(perm) ? img : not;
sb.append(src.charAt(step));
step++;
}
return sb.toString();
}
public String getPerms() {
return perms;
}
public void setPerms(String perms) {
this.perms = perms;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getOwner() {
return owner;
}
public void setOwner(String owner) {
this.owner = owner;
}
public String getGroup() {
return group;
}
public void setGroup(String group) {
this.group = group;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
public long getMtimeMillis() {
return mtime;
}
public void setMtimeMillis(long mtime) {
this.mtime = mtime;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
private String name;
public boolean contains(String content) {
if (!Files.isRegularFile(_path)) {
return false;
}
try {
String s = Files.readString(this._path);
return s.matches("(?s)" + content);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

114
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspaceView.java
View File

@ -1,114 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.joda.time.Period;
import org.joda.time.format.PeriodFormatter;
import org.joda.time.format.PeriodFormatterBuilder;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
public class WorkspaceView {
private final static PeriodFormatter pf = new PeriodFormatterBuilder()
.appendWeeks().appendSuffix(" week", " weeks").appendSeparator(" ")
.appendDays().appendSuffix(" day", " days").appendSeparator(" ")
.appendHours().appendSuffix("H")
.appendMinutes().appendSuffix("M")
.appendSeconds().appendSuffix("S")
.toFormatter();
private final Path workspaceRoot;
private Summary summary;
@JsonProperty("ls")
private List<WorkspaceItemView> listing = null;
public WorkspaceView(Path workspaceRoot) {
this.workspaceRoot = workspaceRoot;
}
public String getName() {
return workspaceRoot.getFileName().toString();
}
@JsonProperty("modified")
public long getModified() {
try {
return Files.getLastModifiedTime(workspaceRoot).toMillis();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@JsonProperty("summary")
public Summary getSummary() {
if (this.summary == null) {
Summary v = new Summary(this.workspaceRoot);
try {
Files.walkFileTree(this.workspaceRoot, v);
this.summary = v;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
return this.summary;
}
public void setListing(List<WorkspaceItemView> listing) {
this.listing = listing;
}
public final static class Summary extends SimpleFileVisitor<Path> {
private final Path root;
public long total_bytes = 0L;
public long total_files = 0L;
public long last_changed_epoch = Long.MIN_VALUE;
public String last_changed_filename = "";
public String getLast_changed_ago() {
int millis = (int) (System.currentTimeMillis() - last_changed_epoch);
Period period = Period.millis(millis);
return pf.print(period.normalizedStandard());
}
public Summary(Path root) {
this.root = root;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
total_bytes += Files.size(file);
total_files++;
long millis= Files.getLastModifiedTime(file).toMillis();
if (last_changed_epoch <millis) {
last_changed_epoch = millis;
last_changed_filename = root.relativize(file).toString();
}
return super.visitFile(file, attrs);
}
}
}

44
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/WorkspacesView.java
View File

@ -1,44 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.rest.transfertypes;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
public class WorkspacesView {
private final Path workspacesRoot;
public WorkspacesView(Path workspacesRoot) {
this.workspacesRoot = workspacesRoot;
}
public List<String> getWorkspaces() {
List<String> workspaces = new ArrayList<>();
try (DirectoryStream<Path> paths = Files.newDirectoryStream(workspacesRoot)) {
for (Path path : paths) {
workspaces.add(path.toString());
}
} catch (IOException e) {
throw new RuntimeException(e);
}
return workspaces;
}
}

29
engine-rest/src/main/java/io/nosqlbench/engine/rest/transfertypes/package-info.java
View File

@ -1,29 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Types in this package are meant to provide a mapping
* between internal state and external views. Specifically,
* these types should only include details which are meant to
* be shared by endpoints. This can be achieved by wrapping
* internal state and exposing only the visible properties, or
* it can be done by implementing types which are built from
* common internal types.
*
* Service objects in the services package should only provide
* primitive values or the view types from this package.
*/
package io.nosqlbench.engine.rest.transfertypes;

1789
engine-rest/src/main/resources/stargate.yaml
View File

File diff suppressed because it is too large Load Diff

57
engine-rest/src/test/resources/log4j2-test.xml
View File

@ -1,57 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2022 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<Configuration status="${project.test_logstatuslevel}" strict="true" name="XMLConfigTest"
packages="org.apache.logging.log4j.test">
<Filter type="ThresholdFilter" level="info"/>
<Appenders>
<Appender type="Console" name="STDOUT">
<Layout type="PatternLayout" pattern="%7r %-5level [%t] %-12logger{0} %msg%n%throwable"/>
<Filters>
<Filter type="MarkerFilter" marker="FLOW" onMatch="DENY" onMismatch="NEUTRAL"/>
<Filter type="MarkerFilter" marker="EXCEPTION" onMatch="DENY" onMismatch="ACCEPT"/>
</Filters>
</Appender>
<Appender type="Console" name="FLOW">
<Layout type="PatternLayout" pattern="%C{1}.%M %m %ex%n"/><!-- class and line number -->
<Filters>
<Filter type="MarkerFilter" marker="FLOW" onMatch="ACCEPT" onMismatch="NEUTRAL"/>
<Filter type="MarkerFilter" marker="EXCEPTION" onMatch="ACCEPT" onMismatch="DENY"/>
</Filters>
</Appender>
<Appender type="File" name="APPSLOG" fileName="docs/apps.log">
<Layout type="PatternLayout">
<Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
</Layout>
</Appender>
</Appenders>
<Loggers>
<Logger name="io.nosqlbench.docsys" level="info" additivity="false">
<AppenderRef ref="APPSLOG"/>
</Logger>
<Root level="${project.testlevel}">
<AppenderRef ref="STDOUT"/>
</Root>
</Loggers>
</Configuration>

103
mvn-defaults/pom.xml
View File

@ -70,7 +70,6 @@
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
@ -78,25 +77,21 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy</artifactId>
<version>4.0.13</version>
</dependency>
<dependency>
<groupId>org.snakeyaml</groupId>
<artifactId>snakeyaml-engine</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
<version>4.7.4</version>
</dependency>
<!-- due to https://github.com/oshi/oshi/issues/1289 -->
<dependency>
<groupId>net.java.dev.jna</groupId>
@ -108,44 +103,37 @@
<artifactId>jna-platform</artifactId>
<version>5.13.0</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<version>5.9.0-M1</version>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mpierce.metrics.reservoir</groupId>
<artifactId>hdrhistogram-metrics-reservoir</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>org.hdrhistogram</groupId>
<artifactId>HdrHistogram</artifactId>
<version>2.1.12</version>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-graphite</artifactId>
<version>4.2.19</version>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>4.2.19</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.10.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-core</artifactId>
@ -171,7 +159,6 @@
<artifactId>commons-statistics-distribution</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
@ -182,53 +169,65 @@
<artifactId>jmh-generator-annprocess</artifactId>
<version>1.36</version>
</dependency>
<dependency>
<groupId>com.mitchtalmadge</groupId>
<artifactId>ascii-data</artifactId>
<version>1.4.0</version>
</dependency>
<!-- For CQL compression option -->
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>1.4.1</version>
</dependency>
<!-- For CQL compression option -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.10.1</version>
</dependency>
<dependency>
<groupId>com.datastax.oss</groupId>
<artifactId>java-driver-query-builder</artifactId>
<version>4.16.0</version>
</dependency>
<dependency>
<groupId>org.snakeyaml</groupId>
<artifactId>snakeyaml-engine</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.10.1</version>
</dependency>
<dependency>
<groupId>com.esri.geometry</groupId>
<artifactId>esri-geometry-api</artifactId>
<version>2.2.4</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.94.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-epoll</artifactId>
<version>4.1.51.Final</version>
<classifier>linux-x86_64</classifier>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-kqueue</artifactId>
<version>4.1.46.Final</version>
<classifier>linux-x86_64</classifier>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-haproxy</artifactId>
<version>4.1.86.Final</version>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-api</artifactId>
@ -240,11 +239,40 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>1.69</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.15.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>3.6.4</version>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>tinkergraph-gremlin</artifactId>
<version>3.6.4</version>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-core</artifactId>
<version>3.3.0</version>
<version>3.3.2</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@ -252,13 +280,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.10.0</version>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-transport-okhttp</artifactId>
@ -269,9 +295,7 @@
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java</artifactId>
@ -284,55 +308,46 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core-java11</artifactId>
<version>6.4.2</version>
<version>6.4.3</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.481</version>
<version>1.12.498</version>
</dependency>
<dependency>
<groupId>com.elega9t</groupId>
<artifactId>number-to-words</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.greenrobot</groupId>
<artifactId>essentials</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>com.squareup</groupId>
<artifactId>javapoet</artifactId>
<version>1.13.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.12.5</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
@ -341,44 +356,38 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
<version>1.16.0</version>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
<version>2.4.15.Final</version>
<version>2.5.0.Final</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>4.13.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.21</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>2.9.8</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>2.3.0.1</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>2.4.0-b180830.0438</version>
</dependency>
<!-- graalvm -->
<dependency>
<groupId>org.graalvm.sdk</groupId>
@ -388,7 +397,7 @@
<dependency>
<groupId>org.graalvm.js</groupId>
<artifactId>js</artifactId>
<version>22.3.2</version>
<version>22.3.3</version>
<scope>runtime</scope>
</dependency>
<dependency>

123
nb-api/src/main/java/io/nosqlbench/api/engine/util/SSLKsFactory.java
View File

@ -22,9 +22,7 @@ import org.apache.logging.log4j.Logger;
import javax.net.ServerSocketFactory;
import javax.net.SocketFactory;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.*;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.security.KeyFactory;
@ -45,11 +43,31 @@ public class SSLKsFactory implements NBMapConfigurable {
private static final SSLKsFactory instance = new SSLKsFactory();
private static final Pattern CERT_PATTERN = Pattern.compile("-+BEGIN\\s+.*CERTIFICATE[^-]*-+(?:\\s|\\r|\\n){1,10}([a-z0-9+/=\\r\\n]+)-+END\\s+.*CERTIFICATE[^-]*-+", Pattern.CASE_INSENSITIVE);
private static final Pattern KEY_PATTERN = Pattern.compile("-+BEGIN\\s+.*PRIVATE\\s+KEY[^-]*-+(?:\\s|\\r|\\n){1,10}([a-z0-9+/=\\r\\n]+)-+END\\s+.*PRIVATE\\s+KEY[^-]*-+", Pattern.CASE_INSENSITIVE);
private static final Pattern CERT_PATTERN = Pattern.compile("-+BEGIN\\s+.*CERTIFICATE[^-]*-+(?:\\s|\\r|\\n)" +
"{1,10}([a-z0-9+/=\\r\\n]+)-+END\\s+.*CERTIFICATE[^-]*-+", Pattern.CASE_INSENSITIVE);
private static final Pattern KEY_PATTERN = Pattern.compile("-+BEGIN\\s+.*PRIVATE\\s+KEY[^-]*-+(?:\\s|\\r|\\n)" +
"{1,10}([a-z0-9+/=\\r\\n]+)-+END\\s+.*PRIVATE\\s+KEY[^-]*-+", Pattern.CASE_INSENSITIVE);
public static final String SSL = "ssl";
public static final String DEFAULT_TLSVERSION = "TLSv1.2";
private static final TrustManager[] trustAllCerts = new TrustManager[]{
new X509TrustManager() {
@Override
public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType) {
}
@Override
public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType) {
}
@Override
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return new java.security.cert.X509Certificate[]{};
}
}
};
/**
* Consider: https://gist.github.com/artem-smotrakov/bd14e4bde4d7238f7e5ab12c697a86a3
*/
@ -70,6 +88,7 @@ public class SSLKsFactory implements NBMapConfigurable {
public SocketFactory createSocketFactory(NBConfiguration cfg) {
SSLContext context = getContext(cfg);
if (context == null) {
throw new IllegalArgumentException("SSL is not enabled.");
}
@ -81,18 +100,26 @@ public class SSLKsFactory implements NBMapConfigurable {
if (sslParam.isPresent()) {
String tlsVersion = cfg.getOptional("tlsversion").orElse(DEFAULT_TLSVERSION);
KeyStore keyStore;
KeyStore keyStore = null;
char[] keyPassword = null;
KeyStore trustStore;
KeyStore trustStore = null;
boolean activeSSLValidation = true;
if (cfg.getOptional("sslValidation").isPresent() && cfg.getOptional("sslValidation")
.get().equals("false")) {
logger.warn("sslValidation=false, skipping SSL validation. " +
"THIS OPTION SHOULD ONLY BE TESTING AND NON-PROD ENVIRONMENTS");
activeSSLValidation = false;
}
if (sslParam.get().equals("jdk")) {
final char[] keyStorePassword = cfg.getOptional("kspass")
.map(String::toCharArray)
.orElse(null);
.map(String::toCharArray)
.orElse(null);
keyPassword = cfg.getOptional("keyPassword", "keypassword")
.map(String::toCharArray)
.orElse(keyStorePassword);
.map(String::toCharArray)
.orElse(keyStorePassword);
keyStore = cfg.getOptional("keystore").map(ksPath -> {
try {
@ -105,9 +132,9 @@ public class SSLKsFactory implements NBMapConfigurable {
trustStore = cfg.getOptional("truststore").map(tsPath -> {
try {
return KeyStore.getInstance(new File(tsPath),
cfg.getOptional("tspass")
.map(String::toCharArray)
.orElse(null));
cfg.getOptional("tspass")
.map(String::toCharArray)
.orElse(null));
} catch (Exception e) {
throw new RuntimeException("Unable to load the truststore: " + e, e);
}
@ -115,8 +142,8 @@ public class SSLKsFactory implements NBMapConfigurable {
} else if (sslParam.get().equals("openssl")) {
try {
CertificateFactory cf = CertificateFactory.getInstance("X.509");
CertificateFactory cf = CertificateFactory.getInstance("X.509");
keyStore = KeyStore.getInstance("JKS");
keyStore.load(null, null);
@ -125,9 +152,8 @@ public class SSLKsFactory implements NBMapConfigurable {
return cf.generateCertificate(is);
} catch (Exception e) {
throw new RuntimeException(
String.format("Unable to load cert from %s: " + e, certFilePath),
e
);
String.format("Unable to load cert from %s: due to:%s ", certFilePath,
e.getMessage()), e);
}
}).orElse(null);
@ -135,37 +161,33 @@ public class SSLKsFactory implements NBMapConfigurable {
keyStore.setCertificateEntry("certFile", cert);
File keyFile = cfg.getOptional("keyfilepath").map(File::new)
.orElse(null);
.orElse(null);
if (keyFile != null) {
try {
keyPassword = cfg.getOptional("keyPassword", "keypassword")
.map(String::toCharArray)
.orElse("temp_key_password".toCharArray());
.map(String::toCharArray)
.orElse("temp_key_password".toCharArray());
KeyFactory kf = KeyFactory.getInstance("RSA");
PrivateKey key = kf.generatePrivate(new PKCS8EncodedKeySpec(loadKeyFromPem(keyFile)));
keyStore.setKeyEntry("key", key, keyPassword,
cert != null ? new Certificate[]{cert} : null);
cert != null ? new Certificate[]{cert} : null);
} catch (Exception e) {
throw new RuntimeException(String.format("Unable to load key from %s: " + e,
keyFile),
e);
throw new RuntimeException(String.format("Unable to load key from: %s due to: %s ",
keyFile, e.getMessage()), e);
}
}
trustStore = cfg.getOptional("caCertFilePath", "cacertfilepath").map(caCertFilePath -> {
try (InputStream is = new FileInputStream(new File(caCertFilePath))) {
try (InputStream is = new FileInputStream(caCertFilePath)) {
KeyStore ts = KeyStore.getInstance("JKS");
ts.load(null, null);
Certificate caCert = cf.generateCertificate(is);
ts.setCertificateEntry("caCertFile", caCert);
ts.setCertificateEntry("caCertFile", cf.generateCertificate(is));
return ts;
} catch (Exception e) {
throw new RuntimeException(String.format("Unable to load caCert from %s: " + e,
caCertFilePath),
e);
throw new RuntimeException(String.format("Unable to load caCert from: %s due to: %s",
caCertFilePath, e.getMessage()), e);
}
}).orElse(null);
@ -183,20 +205,26 @@ public class SSLKsFactory implements NBMapConfigurable {
kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(keyStore, keyPassword);
} catch (Exception e) {
throw new RuntimeException("Unable to init KeyManagerFactory. Please check password and location: " + e, e);
throw new RuntimeException("Unable to init KeyManagerFactory. Please check password and location: "
+ e.getMessage(), e);
}
TrustManagerFactory tmf;
try {
tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore != null ? trustStore : keyStore);
} catch (Exception e) {
throw new RuntimeException("Unable to init TrustManagerFactory: " + e, e);
throw new RuntimeException("Unable to init TrustManagerFactory: " + e.getMessage(), e);
}
try {
SSLContext sslContext = SSLContext.getInstance(tlsVersion);
sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
if (!activeSSLValidation) {
sslContext.init(kmf.getKeyManagers(), trustAllCerts, new SecureRandom());
} else {
sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
}
return sslContext;
} catch (Exception e) {
throw new RuntimeException(e);
@ -229,18 +257,19 @@ public class SSLKsFactory implements NBMapConfigurable {
public NBConfigModel getConfigModel() {
return ConfigModel.of(SSLKsFactory.class,
Param.optional("ssl")
.setDescription("Enable ssl and set the mode")
.setRegex("jdk|openssl"),
Param.defaultTo("tlsversion", DEFAULT_TLSVERSION),
Param.optional("kspass"),
Param.optional("keyPassword"),
Param.optional("keystore"),
Param.optional("truststore"),
Param.optional("tspass"),
Param.optional(List.of("keyFilePath","keyfilepath")),
Param.optional("caCertFilePath"),
Param.optional("certFilePath")
Param.optional("ssl")
.setDescription("Enable ssl and set the mode")
.setRegex("jdk|openssl"),
Param.defaultTo("tlsversion", DEFAULT_TLSVERSION),
Param.optional("kspass"),
Param.optional("keyPassword"),
Param.optional("keystore"),
Param.optional("truststore"),
Param.optional("tspass"),
Param.optional(List.of("keyFilePath", "keyfilepath")),
Param.optional("caCertFilePath"),
Param.optional("certFilePath"),
Param.optional("sslValidation")
).asReadOnly();
}
}

6
virtdata-api/pom.xml
View File

@ -43,24 +43,19 @@
<version>${revision}</version>
<artifactId>nb-api</artifactId>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lang</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>com.squareup</groupId>
<artifactId>javapoet</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-core</artifactId>
@ -76,7 +71,6 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-legacy-exception</artifactId>
<version>4.0-beta1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

42
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/unary_int/SignalPID.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.unary_int;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
/**
* This function provides the current NB process identifier.
* Primarily used when NB is used as a signal agent.
*/
@ThreadSafeMapper
@Categories({Category.general})
public class SignalPID implements LongFunction<Long> {
private final Long pid;
public SignalPID() {
this.pid = ProcessHandle.current().pid();
}
@Override
public Long apply(long i1) {
return this.pid;
}
}

25
engine-rest/src/test/java/io/nosqlbench/engine/rest/services/openapi/OpenApiLoaderTest.java → virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/tests/long_long/SignalPIDTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,24 +14,23 @@
* limitations under the License.
*/
package io.nosqlbench.engine.rest.services.openapi;
package io.nosqlbench.virtdata.library.basics.tests.long_long;
import io.nosqlbench.virtdata.library.basics.shared.unary_int.SignalPID;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
public class OpenApiLoaderTest {
private final static Logger logger = LogManager.getLogger(OpenApiLoaderTest.class);
import static org.assertj.core.api.Assertions.assertThat;
public class SignalPIDTest {
private final static Logger logger = LogManager.getLogger(SignalPIDTest.class);
@Test
public void testYamlGenerator() {
String openidpath = "stargate.yaml";
String filterJson = "{\n" +
"'POST /api/rest/v1/auth' : {}\n" +
"}\n";
String result = OpenApiLoader.generateWorkloadFromFilepath(openidpath, filterJson);
logger.debug(result);
public void testSignalPIDIdentity() {
final Long currentProcessId = ProcessHandle.current().pid();
assertThat(new SignalPID().apply(1L)).isEqualTo(currentProcessId);
assertThat(new SignalPID().apply(2L)).isEqualTo(currentProcessId);
}
}