diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb38dda5a..90fc6ef8e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -98,11 +98,11 @@ jobs: current_branch=$(git rev-parse --abbrev-ref HEAD) if [[ ${current_branch} == *"main"* ]] then + echo "PRERELEASE=false" >> $GITHUB_ENV + echo "DOCKER_TAGS=nosqlbench/nosqlbench:latest,nosqlbench/nosqlbench:4x,nosqlbench/nosqlbench:${{ env.RELEASE_VERSION }}" >> $GITHUB_ENV + else echo "PRERELEASE=true" >> $GITHUB_ENV - echo "DOCKER_TAGS=nosqlbench/nosqlbench:${{ env.RELEASE_VERSION }}" >> $GITHUB_ENV - #else - # echo "PRERELEASE=false" >> $GITHUB_ENV - # echo "DOCKER_TAGS=nosqlbench/nosqlbench:latest,nosqlbench/nosqlbench:${{ env.RELEASE_VERSION }}" >> $GITHUB_ENV + echo "DOCKER_TAGS=nosqlbench/nosqlbench:nb4preview,nosqlbench/nosqlbench:${{ env.RELEASE_VERSION }}" >> $GITHUB_ENV fi - name: prepare Maven release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index febee53eb..208e1da06 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,7 +113,8 @@ are eager to get it into the hands of users who need it. contributors alike. This takes on many forms. It is essential that we keep looking for ways to connect the NoSQLBench community, doing more of what works and less of what doesn't. If you want to help with community - development, please join our slack channel and raise your hand! + development, please join our + [discord server](https://discord.gg/dBHRakusMN) and raise your hand! diff --git a/README.md b/README.md index 3a6588b7f..ebe3b98b1 100644 --- a/README.md +++ b/README.md @@ -54,15 +54,12 @@ armor, so that they feel capable of doing most things autonomously. Please keep support form our community, and help us find those places where the docs are lacking. Maybe you can help us by adding some missing docs! -### NoSQLBench Slack +### NoSQLBench Discord Server -There is a new slack channel at nosqlbench.slack.com. In order to access the slack channel, you'll need an invite. You -can get that -[here](https://join.slack.com/t/nosqlbench/shared_invite/zt-hvavu1tu-CLX_w7plZrv_A8kDBCw_3A), -which will send you an invite in email: Slack Invite. This is just a simple google form that automates the invite -process. - -Please join it if you are a new or existing NoSQLBench user and help us get it going! +We have a discord server. This is where users and developers can discuss +anything about NoSQLBench and support each other. +Please [join us](https://discord.gg/dBHRakusMN) there if you are a new +user of NoSQLBench! ## Contributing diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 840503281..19db059e3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,26 +1 @@ -- dbdbf6e8 (HEAD -> main, origin/main) merge fixes -- 7ba0ca14 adapt pulsar driver to updated API naming -- 154876a3 resolve split version break after jetty security fix -- 304cba2f jetty security update -- 22ab5841 Minor document correction for clarification. -- 5639a544 post-merge fixups -- af4452be include block naming in opdef test -- ea8702c9 basic doc updates and cleanups -- 345877c0 allow both count and counter in old and new versions of error handler -- 63158fcd support mapped blocks -- f1a15230 Minor code change - change method name from IsBoolean() to isBoolean() -- 3ad3cc6b Minor document update -- 58a93bd7 Document update -- adf1ea03 Pulsar Batch API, Async API (partial), yaml file structure, -- 27856703 update cqld4 pom versions -- ada7aaad minor opdef fix -- 0880f0cd pulsar merge fixes -- 8b0f4615 cqld4 partial progress -- b3084fdd Pulsar Reader API with Avro schema support -- b0da4a14 Pulsar Consumer API with Avro schema support -- 3543dd8b support counter error handler name for cross-compatibility -- d59f3f0d remove temporary typo warning -- c3f01b12 make deprecation warning for v1 yaml format stronger -- eb784a13 backport question anchors with deprecation warning -- 50c03632 document op dispenser as OpDispenser type -- a27ea6cd reset cqld4 for reinit +- 59a5ce89 Workarond for Pulsar bug diff --git a/docsys/pom.xml b/docsys/pom.xml index 625cc27e4..fc6c8cfc5 100644 --- a/docsys/pom.xml +++ b/docsys/pom.xml @@ -9,7 +9,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -18,7 +18,7 @@ io.nosqlbench nb-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT @@ -31,75 +31,90 @@ org.eclipse.jetty jetty-server + 11.0.1 org.eclipse.jetty jetty-servlets - 9.4.35.v20201120 + 11.0.1 org.eclipse.jetty jetty-servlet + 11.0.1 org.eclipse.jetty jetty-rewrite + 11.0.1 + org.glassfish.jersey.core jersey-server + 3.0.1 + org.glassfish.jersey.core jersey-common + 3.0.1 org.glassfish.jersey.containers jersey-container-jetty-servlet + 3.0.1 org.glassfish.jersey.inject jersey-hk2 + 3.0.1 org.apache.commons commons-compress + 1.20 - - javax.ws.rs - javax.ws.rs-api - + org.glassfish.jersey.media jersey-media-json-jackson + 3.0.1 com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider + 2.12.2 + + + + com.sun.xml.bind + jaxb-core + 2.3.0.1 + + + com.sun.xml.bind + jaxb-impl + 2.4.0-b180830.0438 javax.xml.bind jaxb-api - - - com.sun.xml.bind - jaxb-core - - - com.sun.xml.bind - jaxb-impl + 2.4.0-b180830.0359 javax.activation activation + 1.1.1 + io.nosqlbench virtdata-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java b/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java index acc703ff2..f19946a37 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServer.java @@ -5,6 +5,8 @@ import io.nosqlbench.docsys.api.Docs; import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.docsys.handlers.FavIconHandler; import io.nosqlbench.nb.api.spi.SimpleServiceLoader; +import jakarta.servlet.DispatcherType; +import jakarta.servlet.ServletRegistration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.eclipse.jetty.rewrite.handler.RewriteHandler; @@ -26,8 +28,6 @@ import org.glassfish.jersey.server.ServerProperties; import org.glassfish.jersey.server.model.ResourceMethod; import org.glassfish.jersey.servlet.ServletContainer; -import javax.servlet.DispatcherType; -import javax.servlet.ServletRegistration; import java.awt.*; import java.net.MalformedURLException; import java.net.URI; diff --git a/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocServerStatusEndpoint.java b/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocServerStatusEndpoint.java index 04f3eff7f..d16b088b4 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocServerStatusEndpoint.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocServerStatusEndpoint.java @@ -1,18 +1,17 @@ package io.nosqlbench.docsys.endpoints; +import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.docsys.core.NBWebServer; import io.nosqlbench.nb.annotations.Service; -import io.nosqlbench.docsys.api.WebServiceObject; -import org.apache.logging.log4j.Logger; +import jakarta.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Configuration; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; - -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Configuration; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; +import org.apache.logging.log4j.Logger; @Service(value = WebServiceObject.class, selector = "docserver-status") @Singleton diff --git a/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocsysMarkdownEndpoint.java b/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocsysMarkdownEndpoint.java index eff4dd873..a4a544927 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocsysMarkdownEndpoint.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocsysMarkdownEndpoint.java @@ -7,13 +7,13 @@ import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.docsys.core.DocsysPathLoader; import io.nosqlbench.docsys.core.PathWalker; import io.nosqlbench.nb.annotations.Service; +import jakarta.inject.Singleton; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Singleton; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -128,13 +128,6 @@ public class DocsysMarkdownEndpoint implements WebServiceObject { return list; } -// @GET -// @Path("file") -// @Produces(MediaType.TEXT_PLAIN) -// public String getFileByPath(@QueryParam("path") String pathspec) { -// return getFile(pathspec); -// } -// /** * @param pathspec the path as known to the manifest * @return The contents of a file @@ -153,25 +146,6 @@ public class DocsysMarkdownEndpoint implements WebServiceObject { } } -// private String getFile(String pathspec) { -// pathspec = URLDecoder.decode(pathspec, StandardCharsets.UTF_8); -// for (java.nio.file.Path path : enabled.getPaths()) { -// java.nio.file.Path resolved = path.resolve(pathspec); -// if (Files.isDirectory(resolved)) { -// throw new RuntimeException("Path is a directory: '" + pathspec + "'"); -// } -// if (Files.exists(resolved)) { -// try { -// String content = Files.readString(resolved, StandardCharsets.UTF_8); -// return content; -// } catch (IOException e) { -// throw new RuntimeException(e); -// } -// } -// } -// throw new RuntimeException("Unable to find any valid file at '" + pathspec + "'"); -// } - public java.nio.file.Path findPath(String pathspec) { pathspec = URLDecoder.decode(pathspec, StandardCharsets.UTF_8); for (java.nio.file.Path path : enabled.getPaths()) { diff --git a/docsys/src/main/java/io/nosqlbench/docsys/handlers/EndpointsHandler.java b/docsys/src/main/java/io/nosqlbench/docsys/handlers/EndpointsHandler.java index dc7ca90e3..a9ebbb44e 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/handlers/EndpointsHandler.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/handlers/EndpointsHandler.java @@ -1,12 +1,12 @@ package io.nosqlbench.docsys.handlers; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.servlet.ServletHandler; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; public class EndpointsHandler extends ServletHandler { diff --git a/docsys/src/main/java/io/nosqlbench/docsys/handlers/FavIconHandler.java b/docsys/src/main/java/io/nosqlbench/docsys/handlers/FavIconHandler.java index 659004bd8..752501ed5 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/handlers/FavIconHandler.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/handlers/FavIconHandler.java @@ -1,17 +1,17 @@ package io.nosqlbench.docsys.handlers; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.util.IO; import org.eclipse.jetty.util.resource.Resource; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.net.URL; import java.nio.file.Path; diff --git a/docsys/src/test/java/io/nosqlbench/docsys/core/TestServlet1.java b/docsys/src/test/java/io/nosqlbench/docsys/core/TestServlet1.java index 7c9040882..ecefd7f38 100644 --- a/docsys/src/test/java/io/nosqlbench/docsys/core/TestServlet1.java +++ b/docsys/src/test/java/io/nosqlbench/docsys/core/TestServlet1.java @@ -1,14 +1,14 @@ package io.nosqlbench.docsys.core; import io.nosqlbench.docsys.api.WebServiceObject; -import org.apache.logging.log4j.Logger; +import jakarta.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/driver-cockroachdb/pom.xml b/driver-cockroachdb/pom.xml index 3f0b20262..540d43c99 100644 --- a/driver-cockroachdb/pom.xml +++ b/driver-cockroachdb/pom.xml @@ -5,7 +5,7 @@ io.nosqlbench mvn-defaults - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -21,7 +21,7 @@ io.nosqlbench driver-jdbc - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT org.postgresql diff --git a/driver-cql-shaded/pom.xml b/driver-cql-shaded/pom.xml index b76316e58..e1460962c 100644 --- a/driver-cql-shaded/pom.xml +++ b/driver-cql-shaded/pom.xml @@ -4,7 +4,7 @@ io.nosqlbench mvn-defaults - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java index fb0c97960..460989f43 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivity.java @@ -144,18 +144,19 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef return session; } - private String canonicalizeBindings(String input) { + // for testing + public static String canonicalizeBindings(String input) { StringBuilder sb = new StringBuilder(); Pattern questionPattern = Pattern.compile("\\?(?\\w+)"); Matcher matcher = questionPattern.matcher(input); int count = 0; while (matcher.find()) { - matcher.appendReplacement(sb, matcher.group("arg")); + matcher.appendReplacement(sb, "{" + matcher.group("arg") + "}"); count++; } matcher.appendTail(sb); if (count > 0) { - logger.warn("You are using a deprecated data binding syntax in '" + input + "'. This is supported in the classic CQL driver," + + logger.warn("You are using deprecated data binding syntax in '" + input + "'. This is supported in the classic CQL driver," + " but it is not recognized by other workloads. Please change to the {standard} binding syntax. The canonical" + " syntax for CQL is rendered automatically."); } @@ -191,7 +192,7 @@ public class CqlActivity extends SimpleActivity implements Activity, ActivityDef for (OpTemplate stmtDef : stmts) { - ParsedStmt parsed = stmtDef.getParsed(this::canonicalizeBindings).orError(); + ParsedStmt parsed = stmtDef.getParsed(CqlActivity::canonicalizeBindings).orError(); boolean prepared = stmtDef.getParamOrDefault("prepared", true); boolean parameterized = stmtDef.getParamOrDefault("parameterized", false); long ratio = stmtDef.getParamOrDefault("ratio", 1); diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/core/CQLSessionCache.java b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/core/CQLSessionCache.java index 7db1c24bd..f46bede03 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/core/CQLSessionCache.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/statements/core/CQLSessionCache.java @@ -1,32 +1,7 @@ package io.nosqlbench.activitytype.cql.statements.core; -import java.io.File; -import java.io.IOException; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import javax.net.ssl.SSLContext; - -import com.datastax.driver.core.Cluster; -import com.datastax.driver.core.NettyOptions; -import com.datastax.driver.core.ProtocolOptions; -import com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions; -import com.datastax.driver.core.Session; -import com.datastax.driver.core.policies.DefaultRetryPolicy; -import com.datastax.driver.core.policies.LoadBalancingPolicy; -import com.datastax.driver.core.policies.LoggingRetryPolicy; -import com.datastax.driver.core.policies.RetryPolicy; -import com.datastax.driver.core.policies.RoundRobinPolicy; -import com.datastax.driver.core.policies.WhiteListPolicy; +import com.datastax.driver.core.*; +import com.datastax.driver.core.policies.*; import com.datastax.driver.dse.DseCluster; import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.haproxy.HAProxyCommand; @@ -45,6 +20,18 @@ import io.nosqlbench.nb.api.errors.BasicError; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import javax.net.ssl.SSLContext; +import java.io.File; +import java.io.IOException; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + public class CQLSessionCache implements Shutdownable { private final static Logger logger = LogManager.getLogger(CQLSessionCache.class); @@ -81,8 +68,8 @@ public class CQLSessionCache implements Shutdownable { String driverType = activityDef.getParams().getOptionalString("cqldriver").orElse("dse"); Cluster.Builder builder = - driverType.toLowerCase().equals("dse") ? DseCluster.builder() : - driverType.toLowerCase().equals("oss") ? Cluster.builder() : null; + driverType.equalsIgnoreCase("dse") ? DseCluster.builder() : + driverType.equalsIgnoreCase("oss") ? Cluster.builder() : null; if (builder == null) { throw new RuntimeException("The driver type '" + driverType + "' is not recognized"); @@ -185,6 +172,15 @@ public class CQLSessionCache implements Shutdownable { .ifPresent(builder::withSpeculativeExecutionPolicy); } + activityDef.getParams().getOptionalString("protocol_version") + .map(String::toUpperCase) + .map(ProtocolVersion::valueOf) + .map(pv -> { + logger.info("protocol_version=>" + pv); + return pv; + }) + .ifPresent(builder::withProtocolVersion); + activityDef.getParams().getOptionalString("socketoptions") .map(sockopts -> { logger.info("socketoptions=>" + sockopts); diff --git a/driver-cql-shaded/src/main/java/io/nosqlbench/endpoints/cql/CqlWorkloadGeneratorEndpoint.java b/driver-cql-shaded/src/main/java/io/nosqlbench/endpoints/cql/CqlWorkloadGeneratorEndpoint.java index a666daf55..dfdb603fc 100644 --- a/driver-cql-shaded/src/main/java/io/nosqlbench/endpoints/cql/CqlWorkloadGeneratorEndpoint.java +++ b/driver-cql-shaded/src/main/java/io/nosqlbench/endpoints/cql/CqlWorkloadGeneratorEndpoint.java @@ -1,14 +1,13 @@ package io.nosqlbench.endpoints.cql; -import io.nosqlbench.generators.cql.lang.CqlWorkloadGen; import io.nosqlbench.docsys.api.WebServiceObject; +import io.nosqlbench.generators.cql.lang.CqlWorkloadGen; import io.nosqlbench.nb.annotations.Service; - -import javax.inject.Singleton; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; +import jakarta.inject.Singleton; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; @Service(value = WebServiceObject.class, selector = "cql-workload-generator") @Path("/services/cql/") diff --git a/driver-cql-shaded/src/main/resources/cql.md b/driver-cql-shaded/src/main/resources/cql.md index 3f2c0e461..83560649a 100644 --- a/driver-cql-shaded/src/main/resources/cql.md +++ b/driver-cql-shaded/src/main/resources/cql.md @@ -62,14 +62,21 @@ activity types. The only option supported for this version is `retrypolicy=logging`, which uses the default retry policy, but with logging added. -- **reconnectpolicy** default: none - Applies a reconnection policy in the driver - Supports either `reconnectpolicy=exponential(minDelayInMs,maxDelayInMs)` or `reconnectpolicy=constant(delayInMs)`. - The driver reconnects using this policy when the entire cluster becomes unavailable. +- **reconnectpolicy** default: none - Applies a reconnection policy in the + driver Supports + either `reconnectpolicy=exponential(minDelayInMs,maxDelayInMs)` + or `reconnectpolicy=constant(delayInMs)`. The driver reconnects using + this policy when the entire cluster becomes unavailable. +- **protocol_version** default: unset, defaults to driver default behavior + - Set the CQL protocol version. Valid values are V1, V2, V3, V4, V5, + DSE_V1, DSE_V2. Protocol is usually auto-negotiated, however, the + initial connection may use a lower protocol to ensure connectivity to + older server versions. If you know you are running on a newer server + version, you can set this to match. -- **pooling** default: none - Applies the connection pooling options - to the policy. - Examples: +- **pooling** default: none - Applies the connection pooling options to + the policy. Examples: - `pooling=4:10` keep between 4 and 10 connections to LOCAL hosts - `pooling=4:10,2:5` diff --git a/driver-cql-shaded/src/test/java/io/nosqlbench/activitytype/cql/core/CqlActivityTest.java b/driver-cql-shaded/src/test/java/io/nosqlbench/activitytype/cql/core/CqlActivityTest.java new file mode 100644 index 000000000..829a11473 --- /dev/null +++ b/driver-cql-shaded/src/test/java/io/nosqlbench/activitytype/cql/core/CqlActivityTest.java @@ -0,0 +1,14 @@ +package io.nosqlbench.activitytype.cql.core; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CqlActivityTest { + + @Test + public void testCanonicalize() { + String cb = CqlActivity.canonicalizeBindings("A ?b C"); + assertThat(cb).isEqualTo("A {b} C"); + } +} diff --git a/driver-cqld4/pom.xml b/driver-cqld4/pom.xml index c1693b909..9124e8010 100644 --- a/driver-cqld4/pom.xml +++ b/driver-cqld4/pom.xml @@ -4,7 +4,7 @@ io.nosqlbench mvn-defaults - 4.15.19-SNAPSHOT + 4.15.20-SNAPSHOT ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench engine-api - 4.15.19-SNAPSHOT + 4.15.20-SNAPSHOT io.nosqlbench drivers-api - 4.15.19-SNAPSHOT + 4.15.20-SNAPSHOT @@ -61,7 +61,7 @@ io.nosqlbench virtdata-lib-basics - 4.15.19-SNAPSHOT + 4.15.20-SNAPSHOT compile diff --git a/driver-cqlverify/pom.xml b/driver-cqlverify/pom.xml index 16f52d85d..b7ab5733d 100644 --- a/driver-cqlverify/pom.xml +++ b/driver-cqlverify/pom.xml @@ -4,7 +4,7 @@ io.nosqlbench mvn-defaults - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -24,13 +24,13 @@ io.nosqlbench driver-cql-shaded - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-diag/pom.xml b/driver-diag/pom.xml index 7d6d06aab..8f3f0a1bc 100644 --- a/driver-diag/pom.xml +++ b/driver-diag/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -20,13 +20,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-dsegraph-shaded/pom.xml b/driver-dsegraph-shaded/pom.xml index 8c5e0b8cd..d004ab62d 100644 --- a/driver-dsegraph-shaded/pom.xml +++ b/driver-dsegraph-shaded/pom.xml @@ -4,7 +4,7 @@ io.nosqlbench mvn-defaults - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-http/pom.xml b/driver-http/pom.xml index bfc56e26a..3301572e2 100644 --- a/driver-http/pom.xml +++ b/driver-http/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java index ad190d94c..5531443cd 100644 --- a/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java +++ b/driver-http/src/main/java/io/nosqlbench/activitytype/cmds/HttpFormatParser.java @@ -72,7 +72,7 @@ public class HttpFormatParser { private final static Pattern DOENCODE = Pattern.compile("(URLENCODE|E)\\[\\[(?.+?)\\]\\]"); - private static String rewriteExplicitSections(String template) { + public static String rewriteExplicitSections(String template) { StringBuilder sb = new StringBuilder(); Matcher matcher = DOENCODE.matcher(template); @@ -86,14 +86,14 @@ public class HttpFormatParser { } - private static String rewriteStaticsOnly(String template) { + public static String rewriteStaticsOnly(String template) { StringBuilder sb = new StringBuilder(); String input = template; Matcher matcher = ParsedTemplate.STANDARD_ANCHOR.matcher(input); int idx = 0; while (matcher.find()) { - String pre = input.substring(0, matcher.start()); + String pre = input.substring(idx, matcher.start()); sb.append(URLEncoder.encode(pre, StandardCharsets.UTF_8)); sb.append(matcher.group()); idx = matcher.end(); diff --git a/driver-http/src/main/resources/activities/baselines/http-iot.yaml b/driver-http/src/main/resources/activities/baselines/http-iot.yaml index dda344b73..f625a647b 100644 --- a/driver-http/src/main/resources/activities/baselines/http-iot.yaml +++ b/driver-http/src/main/resources/activities/baselines/http-iot.yaml @@ -1,4 +1,4 @@ -# nb -v run driver=http yaml=http-iot tags=phase:schema stargate_host=my_stargate_host +# nb -v run driver=http yaml=http-iot tags=phase:schema host=my_stargate_host stargate_host=my_stargate_host auth_token=$AUTH_TOKEN description: | This workload emulates a time-series data model and access patterns. This should be identical to the cql variant except for: @@ -85,7 +85,7 @@ blocks: params: ratio: <> statements: - - main-select: GET /v2/keyspaces/<>/<>?where=E[[{"machine_id":{"$eq":"{machine_id}"}},"sensor_name":{"$eq":"{sensor_name}"},"page-size":{"$eq":"<>"}}]] + - main-select: GET http://<>:<>/v2/keyspaces/<>/<>?where=E[[{"machine_id":{"$eq":"{machine_id}"},"sensor_name":{"$eq":"{sensor_name}"}}]]&page-size=<> Accept: "application/json" X-Cassandra-Request-Id: "{request_id}" X-Cassandra-Token: "<>" @@ -106,7 +106,7 @@ blocks: Content-Type: "application/json" body: | { - "machine_id": "{machine_id}" + "machine_id": "{machine_id}", "sensor_name": "{sensor_name}", "time": "{time}", "sensor_value": "{sensor_value}", diff --git a/driver-http/src/main/resources/activities/baselines/http-keyvalue.yaml b/driver-http/src/main/resources/activities/baselines/http-keyvalue.yaml index 34939cef2..0e75488a6 100644 --- a/driver-http/src/main/resources/activities/baselines/http-keyvalue.yaml +++ b/driver-http/src/main/resources/activities/baselines/http-keyvalue.yaml @@ -1,4 +1,4 @@ -# nb -v run driver=http yaml=http-keyvalue tags=phase:schema stargate_host=my_stargate_host +# nb -v run driver=http yaml=http-keyvalue tags=phase:schema stargate_host=my_stargate_host host=my_stargate_host auth_token=$AUTH_TOKEN description: | This workload emulates a key-value data model and access patterns. This should be identical to the cql variant except for: @@ -7,7 +7,7 @@ description: | scenarios: default: - - run driver=http tags==phase:schema threads==1 cycles==UNDEF + - run driver=cql tags==phase:schema threads==1 cycles==UNDEF - run driver=http tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto - run driver=http tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto bindings: @@ -23,42 +23,21 @@ blocks: - name: schema tags: phase: schema + params: + prepared: false statements: - - create-keyspace: POST http://<>:<>/v2/schemas/keyspaces - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "<>" - Content-Type: "application/json" - body: | - { - "name": "<>" - } + - create-keyspace: | + create keyspace if not exists <> + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '<>'} + AND durable_writes = true; tags: name: create-keyspace - - create-table: POST http://<>:<>/v2/schemas/keyspaces/<>/tables - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "<>" - Content-Type: "application/json" - body: | - { - "name": "<>", - "columnDefinitions": - [ - { - "name": "key", - "typeDefinition": "text" - }, - { - "name": "value", - "typeDefinition": "text" - } - ], - "primaryKey": - { - "partitionKey": ["key"] - } - } + - create-table: | + create table if not exists <>.<> ( + key text, + value text, + PRIMARY KEY (key) + ); tags: name: create-table - name: rampup @@ -84,7 +63,7 @@ blocks: params: ratio: 5 statements: - - main-select: GET /v2/keyspaces/<>/<>?where=E[[{"key":{"$eq":"{rw_key}"}}}]] + - main-select: GET http://<>:<>/v2/keyspaces/<>/<>/{rw_key} Accept: "application/json" X-Cassandra-Request-Id: "{request_id}" X-Cassandra-Token: "<>" @@ -105,7 +84,7 @@ blocks: Content-Type: "application/json" body: | { - "key": "{rw_key}" + "key": "{rw_key}", "value": "{rw_value}" } tags: diff --git a/driver-http/src/main/resources/activities/baselines/http-tabular.yaml b/driver-http/src/main/resources/activities/baselines/http-tabular.yaml index 1e3c08d30..6e8d8dda4 100644 --- a/driver-http/src/main/resources/activities/baselines/http-tabular.yaml +++ b/driver-http/src/main/resources/activities/baselines/http-tabular.yaml @@ -1,4 +1,4 @@ -# nb -v http-tabular rampup-cycles=1E6 main-cycles=1E9 stargate_host=my_stargate_host +# nb -v http-tabular rampup-cycles=1E6 main-cycles=1E9 stargate_host=my_stargate_host host=my_stargate_host auth_token=$AUTH_TOKEN description: | This workload emulates a time-series data model and access patterns. This should be identical to the cql variant except for: @@ -8,7 +8,7 @@ description: | scenarios: default: - - run driver=http tags==phase:schema threads==1 cycles==UNDEF + - run driver=cql tags==phase:schema threads==1 cycles==UNDEF - run driver=http tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto - run driver=http tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto bindings: @@ -31,47 +31,22 @@ blocks: - name: schema tags: phase: schema + params: + prepared: false statements: - - create-keyspace: POST http://<>:<>/v2/schemas/keyspaces - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "<>" - Content-Type: "application/json" - body: | - { - "name": "<>" - } + - create-keyspace: | + create keyspace if not exists <> + WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '<>'} + AND durable_writes = true; tags: name: create-keyspace - - create-table: POST http://<>:<>/v2/schemas/keyspaces/<>/tables - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "<>" - Content-Type: "application/json" - body: | - { - "name": "<>", - "columnDefinitions": - [ - { - "name": "part", - "typeDefinition": "text" - }, - { - "name": "clust", - "typeDefinition": "text" - }, - { - "name": "data", - "typeDefinition": "text" - } - ], - "primaryKey": - { - "partitionKey": ["part"], - "clusteringKey": ["clust"] - } - } + - create-table: | + create table if not exists <>.<> ( + part text, + clust text, + data text, + PRIMARY KEY (part,clust) + ); tags: name: create-table - name: rampup @@ -98,13 +73,13 @@ blocks: params: ratio: 5 statements: - - main-select: GET /v2/keyspaces/<>/<>?where=E[[{"part":{"$eq":"{part_layout}"}},"page-size":{"$eq":"{limit}"}}]] - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "<>" - Content-Type: "application/json" - tags: - name: main-select + - main-select: GET http://<>:<>/v2/keyspaces/<>/<>/{part_read}&page-size={limit} + Accept: "application/json" + X-Cassandra-Request-Id: "{request_id}" + X-Cassandra-Token: "<>" + Content-Type: "application/json" + tags: + name: main-select - name: main-write tags: phase: main @@ -112,16 +87,16 @@ blocks: params: ratio: 5 statements: - - main-write: POST http://<>:<>/v2/keyspaces/<>/<> - Accept: "application/json" - X-Cassandra-Request-Id: "{request_id}" - X-Cassandra-Token: "<>" - Content-Type: "application/json" - body: | - { - "part": "{part_write}" - "clust": "{clust_write}", - "data": "{data_write}" - } - tags: - name: main-write + - main-write: POST http://<>:<>/v2/keyspaces/<>/<> + Accept: "application/json" + X-Cassandra-Request-Id: "{request_id}" + X-Cassandra-Token: "<>" + Content-Type: "application/json" + body: | + { + "part": "{part_write}", + "clust": "{clust_write}", + "data": "{data_write}" + } + tags: + name: main-write diff --git a/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/HttpFormatParserTest.java b/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/HttpFormatParserTest.java new file mode 100644 index 000000000..49b2489af --- /dev/null +++ b/driver-http/src/test/java/io/nosqlbench/activitytype/cmds/HttpFormatParserTest.java @@ -0,0 +1,33 @@ +package io.nosqlbench.activitytype.cmds; + +import org.junit.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class HttpFormatParserTest { + + @Test + public void testSimpleFormat() { + Map map = HttpFormatParser.parseInline("GET http://host:12345/path/{\"v\":{\"name\":\"value\"}"); + assertThat(map).containsAllEntriesOf( + Map.of("method", "GET", "uri", "http://host:12345/path/{\"v\":{\"name\":\"value\"}") + ); + } + + @Test + public void testUrlEncodeBasic() { + Map map2 = HttpFormatParser.parseInline("GET http://host:12345/path/E[[{\"v\":{\"name\":\"value\"}]]"); + assertThat(map2).containsAllEntriesOf( + Map.of("method", "GET", "uri", "http://host:12345/path/%7B%22v%22%3A%7B%22name%22%3A%22value%22%7D")); + } + + @Test + public void testUrlEncodeExceptBindings() { + Map map3 = HttpFormatParser.parseInline("GET http://host:12345/path/E[[{\"v\":{\"{name}\":\"{value}\"}]]"); + assertThat(map3).containsAllEntriesOf( + Map.of("method", "GET", "uri", "http://host:12345/path/%7B%22v%22%3A%7B%22{name}%22%3A%22{value}%22%7D")); + } + +} diff --git a/driver-jdbc/pom.xml b/driver-jdbc/pom.xml index 04624aca7..90ba67376 100644 --- a/driver-jdbc/pom.xml +++ b/driver-jdbc/pom.xml @@ -3,7 +3,7 @@ nosqlbench io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT 4.0.0 @@ -18,7 +18,7 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT compile diff --git a/driver-jmx/pom.xml b/driver-jmx/pom.xml index e6881c717..23f70bc20 100644 --- a/driver-jmx/pom.xml +++ b/driver-jmx/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index 9ab0854d0..7c1536525 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -44,13 +44,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-stdout - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-mongodb/pom.xml b/driver-mongodb/pom.xml index 37b72d6d3..2e031fbf9 100644 --- a/driver-mongodb/pom.xml +++ b/driver-mongodb/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -21,13 +21,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-pulsar/pom.xml b/driver-pulsar/pom.xml index fce51406b..96103072a 100644 --- a/driver-pulsar/pom.xml +++ b/driver-pulsar/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -42,13 +42,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-stdout - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java index c1ff0332d..ffccca6b5 100644 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java +++ b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerMapper.java @@ -38,7 +38,8 @@ public class PulsarConsumerMapper extends PulsarOpMapper { return new PulsarConsumerOp( consumer, clientSpace.getPulsarSchema(), - asyncApi + asyncApi, + clientSpace.getPulsarClientConf().getConsumerTimeoutSeconds() ); } } diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java index d3606a059..85c906944 100644 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java +++ b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/ops/PulsarConsumerOp.java @@ -2,37 +2,61 @@ package io.nosqlbench.driver.pulsar.ops; import io.nosqlbench.driver.pulsar.util.AvroUtil; import io.nosqlbench.driver.pulsar.util.PulsarActivityUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.pulsar.client.api.*; import org.apache.pulsar.common.schema.SchemaType; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + public class PulsarConsumerOp implements PulsarOp { + + private final static Logger logger = LogManager.getLogger(PulsarConsumerOp.class); + private final Consumer consumer; private final Schema pulsarSchema; private final boolean asyncPulsarOp; + private final int timeoutSeconds; - public PulsarConsumerOp(Consumer consumer, Schema schema, boolean asyncPulsarOp) { + public PulsarConsumerOp(Consumer consumer, Schema schema, boolean asyncPulsarOp, int timeoutSeconds) { this.consumer = consumer; this.pulsarSchema = schema; this.asyncPulsarOp = asyncPulsarOp; + this.timeoutSeconds = timeoutSeconds; } public void syncConsume() { try { - Message message = consumer.receive(); + Message message; + if (timeoutSeconds <= 0) { + // wait forever + message = consumer.receive(); + } else { + // we cannot use Consumer#receive(timeout, timeunit) due to + // https://github.com/apache/pulsar/issues/9921 + message = consumer + .receiveAsync() + .get(timeoutSeconds, TimeUnit.SECONDS); + } SchemaType schemaType = pulsarSchema.getSchemaInfo().getType(); if (PulsarActivityUtil.isAvroSchemaTypeStr(schemaType.name())) { - String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition(); - org.apache.avro.generic.GenericRecord avroGenericRecord = - AvroUtil.GetGenericRecord_ApacheAvro(avroDefStr, message.getData()); + if (logger.isDebugEnabled()) { + String avroDefStr = pulsarSchema.getSchemaInfo().getSchemaDefinition(); + org.apache.avro.generic.GenericRecord avroGenericRecord = + AvroUtil.GetGenericRecord_ApacheAvro(avroDefStr, message.getData()); - System.out.println("msg-key=" + message.getKey() + " msg-payload=" + avroGenericRecord.toString()); + logger.debug("msg-key={} msg-payload={}", message.getKey(), avroGenericRecord.toString()); + } } else { - System.out.println("msg-key=" + message.getKey() + " msg-payload=" + new String(message.getData())); + if (logger.isDebugEnabled()) { + logger.debug("msg-key={} msg-payload={}", message.getKey(), new String(message.getData())); + } } consumer.acknowledge(message.getMessageId()); - } catch (PulsarClientException e) { + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java index 5b5814759..2c42b054c 100644 --- a/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java +++ b/driver-pulsar/src/main/java/io/nosqlbench/driver/pulsar/util/PulsarNBClientConf.java @@ -228,6 +228,13 @@ public class PulsarNBClientConf { else return confValue.toString(); } + public int getConsumerTimeoutSeconds() { + Object confValue = getConsumerConfValue("consumer.timeout"); + if (confValue == null) + return -1; // infinite + else + return Integer.parseInt(confValue.toString()); + } public String getConsumerSubscriptionName() { Object confValue = getConsumerConfValue("consumer.subscriptionName"); if (confValue == null) diff --git a/driver-stdout/pom.xml b/driver-stdout/pom.xml index b8987ef3c..a1917dfac 100644 --- a/driver-stdout/pom.xml +++ b/driver-stdout/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-tcp/pom.xml b/driver-tcp/pom.xml index 45553c7e2..9f8d9ee9c 100644 --- a/driver-tcp/pom.xml +++ b/driver-tcp/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -24,19 +24,19 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-stdout - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/driver-web/pom.xml b/driver-web/pom.xml index 350355785..6f3afa6b3 100644 --- a/driver-web/pom.xml +++ b/driver-web/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/drivers-api/pom.xml b/drivers-api/pom.xml index 4b469a743..7cff2d854 100644 --- a/drivers-api/pom.xml +++ b/drivers-api/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench nb-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-userlibs - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-api/pom.xml b/engine-api/pom.xml index d80e0857f..12d30f332 100644 --- a/engine-api/pom.xml +++ b/engine-api/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -23,25 +23,25 @@ io.nosqlbench nb-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench nb-annotations - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-userlibs - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java index dc08c6cf6..f948ea004 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java @@ -35,4 +35,11 @@ public class RawStmtsDocList { public List getStmtsDocs() { return rawStmtsDocList; } + + public String toString() { + int docs = rawStmtsDocList.size(); + int blocks = rawStmtsDocList.stream().map(RawStmtsDoc::getBlocks).mapToInt(List::size).sum(); + long optemplates = rawStmtsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count(); + return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates; + } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java index 73a87d186..2d73f4145 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java @@ -190,16 +190,14 @@ public class CoreMotor implements ActivityDefObserver, Motor, Stoppable { public void run() { try { - strideRateLimiter = activity.getStrideLimiter(); - cycleRateLimiter = activity.getCycleLimiter(); - phaseRateLimiter = activity.getPhaseLimiter(); - + inputTimer = activity.getInstrumentation().getOrCreateInputTimer(); stridesServiceTimer = activity.getInstrumentation().getOrCreateStridesServiceTimer(); stridesResponseTimer = activity.getInstrumentation().getStridesResponseTimerOrNull(); optrackerBlockCounter = activity.getInstrumentation().getOrCreateOpTrackerBlockedCounter(); - - inputTimer = activity.getInstrumentation().getOrCreateInputTimer(); + strideRateLimiter = activity.getStrideLimiter(); + cycleRateLimiter = activity.getCycleLimiter(); + phaseRateLimiter = activity.getPhaseLimiter(); if (slotState.get() == Finished) { diff --git a/engine-cli/pom.xml b/engine-cli/pom.xml index c79132eae..ccfd7cdb0 100644 --- a/engine-cli/pom.xml +++ b/engine-cli/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -23,13 +23,13 @@ io.nosqlbench engine-core - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench engine-docker - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 4903a2237..1dc576cba 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -20,6 +20,7 @@ import io.nosqlbench.engine.core.script.Scenario; import io.nosqlbench.engine.core.script.ScenariosExecutor; import io.nosqlbench.engine.core.script.ScriptParams; import io.nosqlbench.engine.docker.DockerMetricsManager; +import io.nosqlbench.nb.api.SystemId; import io.nosqlbench.nb.api.annotations.Annotation; import io.nosqlbench.nb.api.annotations.Layer; import io.nosqlbench.nb.api.content.Content; @@ -285,15 +286,19 @@ public class NBCLI { System.exit(0); } + logger.info("Running NoSQLBench Version " + new VersionInfo().getVersion()); + + logger.info("client-hardware: " + SystemId.getHostSummary()); + logger.debug("initializing annotators with config:'" + annotatorsConfig + "'"); Annotators.init(annotatorsConfig); Annotators.recordAnnotation( - Annotation.newBuilder() - .session(sessionName) - .now() - .layer(Layer.CLI) - .detail("cli", Strings.join(args, "\n")) - .build() + Annotation.newBuilder() + .session(sessionName) + .now() + .layer(Layer.CLI) + .detail("cli", Strings.join(args, "\n")) + .build() ); if (reportGraphiteTo != null || options.wantsReportCsvTo() != null) { diff --git a/engine-clients/pom.xml b/engine-clients/pom.xml index 32a758d08..ba324caf7 100644 --- a/engine-clients/pom.xml +++ b/engine-clients/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -21,7 +21,7 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-core/pom.xml b/engine-core/pom.xml index d5323ff77..fd5f99220 100644 --- a/engine-core/pom.xml +++ b/engine-core/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -28,13 +28,13 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench drivers-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT @@ -85,7 +85,7 @@ io.nosqlbench engine-clients - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT compile diff --git a/engine-docker/pom.xml b/engine-docker/pom.xml index e0382c36d..49f6bbd72 100644 --- a/engine-docker/pom.xml +++ b/engine-docker/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -20,15 +20,6 @@ - - javax.activation - activation - - - javax.xml.bind - jaxb-api - - io.netty netty-handler @@ -65,7 +56,7 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-docs/pom.xml b/engine-docs/pom.xml index 3142d3948..4ae4d97da 100644 --- a/engine-docs/pom.xml +++ b/engine-docs/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -28,7 +28,7 @@ io.nosqlbench docsys - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-docs/src/main/resources/docs-for-nb/nosqlbench/getting_support.md b/engine-docs/src/main/resources/docs-for-nb/nosqlbench/getting_support.md index a154f8f0d..fe9ee2a1e 100644 --- a/engine-docs/src/main/resources/docs-for-nb/nosqlbench/getting_support.md +++ b/engine-docs/src/main/resources/docs-for-nb/nosqlbench/getting_support.md @@ -5,15 +5,17 @@ weight: 10 # Getting Support -In general, our goals with NoSQLBench are to make the help systems and examples wrap around the users like a suit of -armor, so that they feel capable of doing most things without having to ask for help. Please keep this in mind when -looking for personal support form our community, and help us find those places where the docs are lacking. Maybe you can -help us by adding some missing docs! +In general, our goals with NoSQLBench are to make the help systems and +examples wrap around the users like a suit of armor, so that they feel +capable of doing most things without having to ask for help. Please keep +this in mind when looking for personal support form our community, and +help us find those places where the docs are lacking. Maybe you can help +us by adding some missing docs! ## Built-In Docs -The documentation for NoSQLBench is quite thorough. On the command line, you can see a list of built-in docs with the -command: +The documentation for NoSQLBench is quite thorough. On the command line, +you can see a list of built-in docs with the command: nb help topics @@ -21,37 +23,43 @@ To read any topic, simply use the command: nb help -The documentation system you are looking at right now includes the same docs you can find above already and more. They -are automatically included when NoSQLBench is built. +The documentation system you are looking at right now includes the same +docs you can find above already and more. They are automatically included +when NoSQLBench is built. -## NoSQLBench Slack +## NoSQLBench Discord Server -Please click the -[Invite Link for nosqlbench.slack.com](https://join.slack.com/t/nosqlbench/shared_invite/zt-grrg64g3-6SeVi2jaum0cxp51WnvOVA) -to join us. - -Please join it if you are a new or existing NoSQLBench user and help us get it going! +We have a discord server. This is where users and developers can discuss +anything about NoSQLBench and support each other. +Please [join us](https://discord.gg/dBHRakusMN) there if you are a new +user of NoSQLBench! ## General Feedback These guidelines are mirrored at the -[Submitting Feedback](https://github.com/nosqlbench/nosqlbench/wiki/Submitting-Feedback) wiki page at the nosqlbench -project site, which is also where any `[Submit Feedback]` links should will take you. +[Submitting Feedback](https://github.com/nosqlbench/nosqlbench/wiki/Submitting-Feedback) +wiki page at the nosqlbench project site, which is also where +any `[Submit Feedback]` links should will take you. ## Bug Fixes If you think you have found a bug, please -[file a bug report](https://github.com/nosqlbench/nosqlbench/issues/new?labels=bug). nosqlbench is actively used within -DataStax, and verified bugs will get attention as resources permit. Bugs reports which are more detailed, or bug reports -which include steps to reproduce will get attention first. +[file a bug report](https://github.com/nosqlbench/nosqlbench/issues/new?labels=bug) +. nosqlbench is actively used within DataStax, and verified bugs will get +attention as resources permit. Bugs reports which are more detailed, or +bug reports which include steps to reproduce will get attention first. ## Feature Requests -If you would like to see something in nosqlbench that is not there yet,please -[submit a feature request](https://github.com/nosqlbench/nosqlbench/issues/new?labels=feature). +If you would like to see something in nosqlbench that is not there +yet,please +[submit a feature request](https://github.com/nosqlbench/nosqlbench/issues/new?labels=feature) +. ## Documentation Requests -If you would like to see a specific nosqlbench or testing topic added to the guidebook, please -[request docs content](https://github.com/nosqlbench/nosqlbench/issues/new?labels=docs). +If you would like to see a specific nosqlbench or testing topic added to +the guidebook, please +[request docs content](https://github.com/nosqlbench/nosqlbench/issues/new?labels=docs) +. diff --git a/engine-extensions/pom.xml b/engine-extensions/pom.xml index 5c4323d81..2316369ff 100644 --- a/engine-extensions/pom.xml +++ b/engine-extensions/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -22,7 +22,7 @@ io.nosqlbench engine-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-rest/pom.xml b/engine-rest/pom.xml index 80638201b..119a9b265 100644 --- a/engine-rest/pom.xml +++ b/engine-rest/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -47,7 +47,7 @@ io.nosqlbench engine-cli - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/OpenApiEndpoint.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/OpenApiEndpoint.java index 59c892a1f..57e6fa761 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/OpenApiEndpoint.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/OpenApiEndpoint.java @@ -8,11 +8,11 @@ import io.nosqlbench.nb.annotations.Service; import io.swagger.parser.OpenAPIParser; import io.swagger.parser.v2.SwaggerConverter; import io.swagger.util.Json; +import jakarta.inject.Singleton; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; -import javax.inject.Singleton; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.util.Map; @Service(value = WebServiceObject.class, selector = "openapi") diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java index 272b66d57..c4b0b22ee 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java @@ -13,14 +13,14 @@ import io.nosqlbench.engine.rest.services.WorkspaceFinder; import io.nosqlbench.engine.rest.transfertypes.LiveScenarioView; import io.nosqlbench.engine.rest.transfertypes.RunScenarioRequest; import io.nosqlbench.nb.annotations.Service; +import jakarta.inject.Singleton; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; -import javax.inject.Singleton; -import javax.ws.rs.*; -import javax.ws.rs.core.*; import java.io.CharArrayWriter; import java.io.PrintWriter; import java.util.*; diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ServiceStatusEndpoint.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ServiceStatusEndpoint.java index dd3f86de7..f1b703ea3 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ServiceStatusEndpoint.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ServiceStatusEndpoint.java @@ -4,18 +4,18 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.virtdata.userlibs.apps.docsapp.AutoDocsWebService; +import jakarta.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Configuration; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Configuration; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.util.Map; @Service(value = WebServiceObject.class, selector = "service-status") diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkloadFinderEndpoint.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkloadFinderEndpoint.java index c4b0794a0..750821e93 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkloadFinderEndpoint.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkloadFinderEndpoint.java @@ -6,19 +6,22 @@ import io.nosqlbench.engine.api.scenarios.WorkloadDesc; import io.nosqlbench.engine.rest.services.WorkspaceFinder; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.virtdata.userlibs.apps.docsapp.AutoDocsWebService; +import jakarta.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Configuration; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Configuration; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; @Service(value = WebServiceObject.class, selector = "workload-finder") diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkspacesEndpoint.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkspacesEndpoint.java index 421f51816..effdf5f66 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkspacesEndpoint.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/WorkspacesEndpoint.java @@ -6,13 +6,13 @@ import io.nosqlbench.engine.rest.services.WorkspaceFinder; import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView; import io.nosqlbench.engine.rest.transfertypes.WorkspaceView; import io.nosqlbench.nb.annotations.Service; +import jakarta.inject.Singleton; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Singleton; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.*; import java.nio.ByteBuffer; import java.nio.file.Paths; import java.util.List; diff --git a/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkspaceFinder.java b/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkspaceFinder.java index 2957b2ab3..4012ee6e7 100644 --- a/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkspaceFinder.java +++ b/engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkspaceFinder.java @@ -1,11 +1,11 @@ package io.nosqlbench.engine.rest.services; import io.nosqlbench.engine.rest.transfertypes.WorkspaceView; +import jakarta.ws.rs.core.Configuration; +import jakarta.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.ws.rs.core.Configuration; -import javax.ws.rs.core.MediaType; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.*; diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 571634a1e..0898d1d84 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -3,7 +3,7 @@ io.nosqlbench mvn-defaults - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT pom @@ -23,16 +23,10 @@ 1.9.0 0.61.6 3.0.0-RC1 - 2.9.8 1.11.1 - 1.1.1 - 2.1 2.4.0-b180830.0359 2.3.0.1 2.4.0-b180830.0438 - 2.29 - 2.27 - 9.4.7.v20170914 1.22 2.9.9 5.3.2 @@ -101,13 +95,6 @@ test - - - javax.activation - activation - ${javax.activation.version} - - org.mpierce.metrics.reservoir hdrhistogram-metrics-reservoir @@ -138,12 +125,6 @@ ${commons.text.version} - - - - - - org.openjdk.jmh jmh-core @@ -321,76 +302,16 @@ 4.8 - - org.eclipse.jetty - jetty-server - ${jetty.version} - - - org.eclipse.jetty - jetty-servlets - ${jetty.version} - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - - - org.eclipse.jetty - jetty-rewrite - ${jetty.version} - - - org.glassfish.jersey.core - jersey-server - ${jersey.version} - - - org.glassfish.jersey.core - jersey-common - ${jersey.version} - - - org.glassfish.jersey.containers - jersey-container-jetty-servlet - ${jersey.version} - - - org.glassfish.jersey.inject - jersey-hk2 - ${jersey.version} - - org.apache.commons commons-compress ${commons.compress.version} - - javax.ws.rs - javax.ws.rs-api - ${javax.ws.rs.version} - - - - org.glassfish.jersey.media - jersey-media-json-jackson - ${jersey.media.version} - - - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - ${jackson.jaxrs.version} - - - - javax.xml.bind - jaxb-api - ${jaxb.api.version} + 2.9.8 @@ -539,17 +460,18 @@ - - - always - - **/*Integrated*Test*.java - **/*IntegrationTest.java - **/*IntegrationTests.java - - - - + + + 1 + false + + **/*Integrated*Test*.java + **/*IntegrationTest.java + **/*IntegrationTests.java + + + + diff --git a/nb-annotations/pom.xml b/nb-annotations/pom.xml index 5497d9049..d8d9b6265 100644 --- a/nb-annotations/pom.xml +++ b/nb-annotations/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults diff --git a/nb-api/pom.xml b/nb-api/pom.xml index 0f8fbdb82..b1fa022e1 100644 --- a/nb-api/pom.xml +++ b/nb-api/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -31,7 +31,7 @@ io.nosqlbench nb-annotations - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/SystemId.java b/nb-api/src/main/java/io/nosqlbench/nb/api/SystemId.java index cef0ada5d..2a6b2d41d 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/SystemId.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/SystemId.java @@ -1,16 +1,21 @@ package io.nosqlbench.nb.api; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; import oshi.hardware.HardwareAbstractionLayer; import oshi.hardware.NetworkIF; import java.util.*; public class SystemId { + public static String getNodeId() { SystemInfo sysinfo = new SystemInfo(); HardwareAbstractionLayer hal = sysinfo.getHardware(); List interfaces = hal.getNetworkIFs(); + Optional first = interfaces.stream() .filter(i -> !i.getName().startsWith("docker" )) .filter(i -> !i.getName().equals("lo" )) @@ -26,11 +31,45 @@ public class SystemId { } return 0; }) - .flatMap(iface -> Arrays.stream(iface.getIPv4addr().clone())) - .filter(addr -> !(addr.startsWith("127." ))) - .findFirst(); - String systemID = first.orElse("UNKNOWN_SYSTEM_ID" ); + .flatMap(iface -> Arrays.stream(iface.getIPv4addr().clone())) + .filter(addr -> !(addr.startsWith("127."))) + .findFirst(); + String systemID = first.orElse("UNKNOWN_SYSTEM_ID"); return systemID; } + public static String getHostSummary() { + SystemInfo sysinfo = new SystemInfo(); + HardwareAbstractionLayer hal = sysinfo.getHardware(); + CentralProcessor p = hal.getProcessor(); + + Gson gson = new GsonBuilder().create(); + + Set ifspeeds = new HashSet<>(); + hal.getNetworkIFs().forEach( + x -> { + long spd = x.getSpeed(); + if (spd < (1024 * 1024 * 1000)) { + ifspeeds.add(String.format("%.0fMib", (double) (spd / (1024 * 1024)))); + } else { + ifspeeds.add(String.format("%.0fGib", (double) (spd / (1024 * 1024 * 1000)))); + } + } + ); + + Map details = Map.of( + "physical-cores", String.valueOf(p.getPhysicalProcessorCount()), + "logical-cores", String.valueOf(p.getLogicalProcessors().size()), + "max-frequency-ghz", String.format("%.2f", (p.getMaxFreq() / 1_000_000_000_000.0d)), + "sockets", String.valueOf(p.getPhysicalPackageCount()), + "processor-name", String.valueOf(p.getProcessorIdentifier().getName()), + "memory-GiB", String.format("%.2f", hal.getMemory().getTotal() / (1024.0 * 1024.0 * 1024.0)), + "heap-max-GiB", String.format("%.2f", Runtime.getRuntime().maxMemory() / (1024.0 * 1024.0 * 1024.0)), + "if-speeds", ifspeeds + + ); + + return gson.toJson(details); + + } } diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java new file mode 100644 index 000000000..2f62bddb0 --- /dev/null +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java @@ -0,0 +1,12 @@ +package io.nosqlbench.nb.api; + +import org.junit.Test; + +public class SystemIdTest { + + @Test + public void testHostInfo() { + String info = SystemId.getHostSummary(); + System.out.println(info); + } +} diff --git a/nb/pom.xml b/nb/pom.xml index 79896bb8e..51907e212 100644 --- a/nb/pom.xml +++ b/nb/pom.xml @@ -5,7 +5,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -24,143 +24,111 @@ io.nosqlbench engine-rest - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench engine-cli - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench engine-docs - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench engine-core - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench engine-extensions - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT - - - - - - io.nosqlbench driver-web - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-kafka - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-stdout - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-diag - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-tcp - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-http - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-jmx - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-dsegraph-shaded - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-cql-shaded - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-cqlverify - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-mongodb - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench driver-pulsar - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT - - - - - - io.nosqlbench driver-cockroachdb - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT - - javax.activation - activation - - - - javax.xml.bind - jaxb-api - - - - - - - - - - - - @@ -200,17 +168,6 @@ - - - - - - - - - - - org.codehaus.mojo exec-maven-plugin @@ -269,7 +226,7 @@ io.nosqlbench driver-mongodb - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/pom.xml b/pom.xml index a9b18cc8d..cf69a8ff1 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT mvn-defaults diff --git a/virtdata-api/pom.xml b/virtdata-api/pom.xml index 2e233c66c..734f9a0ea 100644 --- a/virtdata-api/pom.xml +++ b/virtdata-api/pom.xml @@ -7,7 +7,7 @@ io.nosqlbench mvn-defaults - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -23,14 +23,14 @@ io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT nb-api io.nosqlbench virtdata-lang - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-lang/pom.xml b/virtdata-lang/pom.xml index 071f54a32..78a0cf2c9 100644 --- a/virtdata-lang/pom.xml +++ b/virtdata-lang/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults diff --git a/virtdata-lib-basics/pom.xml b/virtdata-lib-basics/pom.xml index 54c48b99d..1fac38718 100644 --- a/virtdata-lib-basics/pom.xml +++ b/virtdata-lib-basics/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -20,7 +20,7 @@ io.nosqlbench virtdata-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-lib-curves4/pom.xml b/virtdata-lib-curves4/pom.xml index a90cf0f85..bfe236221 100644 --- a/virtdata-lib-curves4/pom.xml +++ b/virtdata-lib-curves4/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -22,13 +22,13 @@ io.nosqlbench virtdata-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-lib-basics - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-lib-random/pom.xml b/virtdata-lib-random/pom.xml index 7cd37c144..1d0677c22 100644 --- a/virtdata-lib-random/pom.xml +++ b/virtdata-lib-random/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -20,13 +20,13 @@ io.nosqlbench virtdata-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-lib-basics - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-lib-realer/pom.xml b/virtdata-lib-realer/pom.xml index 1e5bc8400..e60be5db1 100644 --- a/virtdata-lib-realer/pom.xml +++ b/virtdata-lib-realer/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -20,7 +20,7 @@ io.nosqlbench virtdata-lib-basics - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-realdata/pom.xml b/virtdata-realdata/pom.xml index deb4df457..7611bc667 100644 --- a/virtdata-realdata/pom.xml +++ b/virtdata-realdata/pom.xml @@ -7,7 +7,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -18,7 +18,7 @@ io.nosqlbench virtdata-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-userlibs/pom.xml b/virtdata-userlibs/pom.xml index ab7b846f6..fe59763d4 100644 --- a/virtdata-userlibs/pom.xml +++ b/virtdata-userlibs/pom.xml @@ -4,7 +4,7 @@ mvn-defaults io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT ../mvn-defaults @@ -18,36 +18,36 @@ io.nosqlbench virtdata-realdata - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-lib-realer - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-api - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench virtdata-lib-random - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT virtdata-lib-basics io.nosqlbench - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT virtdata-lib-curves4 @@ -55,7 +55,7 @@ io.nosqlbench docsys - 4.15.20-SNAPSHOT + 4.15.29-SNAPSHOT diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataService.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataService.java index 8a17db0c8..279bf83b3 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataService.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataService.java @@ -1,17 +1,16 @@ package io.nosqlbench.virtdata.userlibs.apps; +import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.virtdata.core.bindings.DataMapper; import io.nosqlbench.virtdata.core.bindings.ResolverDiagnostics; import io.nosqlbench.virtdata.core.bindings.VirtData; -import io.nosqlbench.docsys.api.WebServiceObject; - -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; +import jakarta.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; @Service(value = WebServiceObject.class, selector = "virtdata") @Singleton diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java index caa358c21..895f70211 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsWebService.java @@ -2,19 +2,19 @@ package io.nosqlbench.virtdata.userlibs.apps.docsapp; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.jaxrs.annotation.JacksonFeatures; -import io.nosqlbench.nb.annotations.Service; -import io.nosqlbench.virtdata.core.bindings.VirtDataDocs; import io.nosqlbench.docsys.api.WebServiceObject; +import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.virtdata.api.processors.DocFuncData; +import io.nosqlbench.virtdata.core.bindings.VirtDataDocs; +import jakarta.inject.Singleton; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import javax.inject.Singleton; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; import java.util.List; import java.util.stream.Collectors;