From ff188d48606df2f5999e0cf8b3a20ec79077a9af Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Fri, 13 Jan 2023 11:57:08 -0600 Subject: [PATCH] actions-for-docs (#919) * updated docs action with version state * script updates * misc doc system updates * Update build.yml remove versions logic, since zip contains version info now. * Update build.yml corrected git add command * update version format * updated docs site url --- .github/workflows/build.yml | 5 +- .gitignore | 1 + .../io/nosqlbench/analysis/RingAnalyzer.java | 2 +- .../cqlgen/core/CGWorkloadExporter.java | 2 +- adapter-cqld4/src/main/resources/cqld4.md | 98 ++++++++++--------- adapter-cqld4/src/main/resources/cqlgen.md | 4 +- .../src/main/resources/dynamodb.md | 2 +- adapter-http/src/main/resources/http.md | 6 +- adapter-mongodb/src/main/resources/mongodb.md | 6 +- adapter-pulsar/src/main/resources/pulsar.md | 13 +-- adapter-s4j/src/main/resources/s4j.md | 20 +--- adapter-stdout/src/main/resources/stdout.md | 36 +++++-- adapter-tcp/src/main/resources/tcpclient.md | 2 +- adapter-tcp/src/main/resources/tcpserver.md | 2 +- .../BundledDriverAdapterDocs.java | 11 +-- .../api/specifications/SpecDocsManifest.java | 39 ++++++++ .../SpecificationDocsManifest.java | 15 ++- .../UniformWorkloadSpecificationTest.java | 4 +- .../YamlSpecValidator.java | 5 +- .../docsys/core/NBWebServerApp.java | 2 +- .../java/io/nosqlbench/engine/cli/NBCLI.java | 2 +- mvn-defaults/pom.xml | 10 ++ .../io/nosqlbench/api/apps/BundledApp.java | 51 ++++++++++ .../nosqlbench/api/apps/BundledAppDocs.java | 42 ++++++++ .../api/docsapi/BundledMarkdownLoader.java | 1 + .../markdown/exporter/MarkdownExporter.java | 2 +- .../nosqlbench/nb/spectest/core/SpecTest.java | 10 ++ nbr/pom.xml | 4 + .../docexporter/BundledMarkdownExporter.java | 2 +- .../docexporter/BundledVersionManifest.java | 34 +++++++ nbr/src/main/resources/nb_version_info.md | 9 ++ .../userlibs/apps/VirtDataMainApp.java | 3 +- .../apps/summarizer/ValueSummarizerApp.java | 2 +- 33 files changed, 327 insertions(+), 120 deletions(-) rename adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/{uniform => docs}/BundledDriverAdapterDocs.java (88%) create mode 100644 adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java rename nb-api/src/main/java/io/nosqlbench/api/spi/BundledApp.java => adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/SpecificationDocsManifest.java (57%) rename adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/{rawyaml => specifications}/UniformWorkloadSpecificationTest.java (94%) rename adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/{rawyaml => specifications}/YamlSpecValidator.java (97%) create mode 100644 nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java create mode 100644 nb-api/src/main/java/io/nosqlbench/api/apps/BundledAppDocs.java create mode 100644 nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java create mode 100644 nbr/src/main/resources/nb_version_info.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 597c47745..b18283993 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,6 +64,7 @@ jobs: needs: build runs-on: ubuntu-20.04 steps: + - name: set git username run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}" @@ -95,12 +96,12 @@ jobs: - name: set CNAME run: | - echo "previewdocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME + echo "builddocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME - name: commit changes run: | cd nosqlbench-build-docs - git add -A + git add exported_docs.zip - name: push changes env: diff --git a/.gitignore b/.gitignore index cfafd1f6c..3b0f858ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +exported_docs.zip .nosqlbench/** .run/** workspaces/** diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzer.java b/adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzer.java index 3b24fc4b1..d1ef02015 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzer.java @@ -16,7 +16,7 @@ package io.nosqlbench.analysis; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.nb.annotations.Service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java index ff8d5a40f..98691c69e 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java @@ -20,7 +20,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.nosqlbench.api.content.Content; import io.nosqlbench.api.content.NBIO; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.cqlgen.binders.Binding; import io.nosqlbench.cqlgen.binders.BindingsAccumulator; import io.nosqlbench.cqlgen.api.BindingsLibrary; diff --git a/adapter-cqld4/src/main/resources/cqld4.md b/adapter-cqld4/src/main/resources/cqld4.md index 49225dd8e..e296e7cfd 100644 --- a/adapter-cqld4/src/main/resources/cqld4.md +++ b/adapter-cqld4/src/main/resources/cqld4.md @@ -1,6 +1,6 @@ -# cqld4 driver +# cqld4 -This is the newly revamped (beta) driver for CQL which uses the DataStax OSS Driver version 4. As +This is the newly revamped driver for CQL which uses the DataStax OSS Driver version 4. As there was a significant restructuring of the APIs between CQL driver 4.x and previous versions, this driver is a clean and separate implementation which aims to use the features of version 4.x of the native driver directly as well as new internal NoSQLBench APIs. @@ -38,20 +38,20 @@ mixing in JSON or remote URLs. Configure directly from a config file, or classpath resource: - # If this isn't found in the file system, the classpath will also be checked. - driverconfig=myconfig.json +```shell +# If this isn't found in the file system, the classpath will also be checked. +nb5 ... driverconfig=myconfig.json +``` Configure directly from JSON: - - driverconfig='{basic.request.timeout:"2 seconds"}' +```shell +nb5 ... driverconfig='{basic.request.timeout:"2 seconds"}' +``` Configure directly form a remote URL: - - driverconfig='http://gist.github.com...' - -Configure from multiple sources: - - driverconfig=myconfig.json +```shell +nb5 ... driverconfig='http://gist.github.com...' +``` ### Basic Cqld4 driver options @@ -109,48 +109,50 @@ classic form have not changed. ## CQLd4 Op Template Examples - ops: +```yaml +ops: - # prepared statement - # allows for parameterization via bindings, and uses prepared statements internally - example-prepared-cql-stmt: - prepared: | - select one, two from buckle.myshoe where ... + # prepared statement + # allows for parameterization via bindings, and uses prepared statements internally + example-prepared-cql-stmt: + prepared: | + select one, two from buckle.myshoe where ... - # prepared statement (verbose form) - example-prepared-cql-stmt-verbose: - type: prepared - stmt: | - select one, two from buckle.myshoe where ... + # prepared statement (verbose form) + example-prepared-cql-stmt-verbose: + type: prepared + stmt: | + select one, two from buckle.myshoe where ... - # simple statement - # allows for parameterization via bindings, but does not use prepared statements internally - example-simple-cql-stmt: - simple: | - select three, four from knock.onthedoor where ... + # simple statement + # allows for parameterization via bindings, but does not use prepared statements internally + example-simple-cql-stmt: + simple: | + select three, four from knock.onthedoor where ... - # raw statement - # pre-renders the statement into a string, with no driver-supervised parameterization - # useful for testing variant DDL where some fields are not parameterizable - # NOTE: the raw form does its best to quote non-literals where needed, but you may - # have to inject single or double quotes in special cases. - example-raw-cql-stmt: - raw: | - create table if not exist {ksname}.{tblname} ... + # raw statement + # pre-renders the statement into a string, with no driver-supervised parameterization + # useful for testing variant DDL where some fields are not parameterizable + # NOTE: the raw form does its best to quote non-literals where needed, but you may + # have to inject single or double quotes in special cases. + example-raw-cql-stmt: + raw: | + create table if not exist {ksname}.{tblname} ... - # gremlin statement using the fluent API, as it would be written in a client application - example-fluent-graph-stmt: - fluent: >- - g.V().hasLabel("device").has("deviceid", UUID.fromString({deviceid})) - # if imports are not specified, the following is auto imported. - # if imports are specified, you must also provide the __ class if needed - imports: - - org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__ + # gremlin statement using the fluent API, as it would be written in a client application + example-fluent-graph-stmt: + fluent: >- + g.V().hasLabel("device").has("deviceid", UUID.fromString({deviceid})) + # if imports are not specified, the following is auto imported. + # if imports are specified, you must also provide the __ class if needed + imports: + - org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__ - # gremlin statement using string API (not recommended) - example-raw-gremlin-stmt: - gremlin: >- - g.V().hasLabel("device").has("deviceid", UUID.fromString('{deviceid})') + # gremlin statement using string API (not recommended) + example-raw-gremlin-stmt: + gremlin: >- + g.V().hasLabel("device").has("deviceid", UUID.fromString('{deviceid})') +``` ## CQL Op Template - Optional Fields diff --git a/adapter-cqld4/src/main/resources/cqlgen.md b/adapter-cqld4/src/main/resources/cqlgen.md index 4dd89f083..4c7a1f207 100644 --- a/adapter-cqld4/src/main/resources/cqlgen.md +++ b/adapter-cqld4/src/main/resources/cqlgen.md @@ -1,4 +1,6 @@ -# cqlgen - A default CQL workload generator +# cqlgen + +cqlgen is a default CQL workload generator With NB5, the cqld4 driver comes with a workload generator that can be used to generate a workload yaml from a CQL schema file. diff --git a/adapter-dynamodb/src/main/resources/dynamodb.md b/adapter-dynamodb/src/main/resources/dynamodb.md index 56c2f8282..90969c2cd 100644 --- a/adapter-dynamodb/src/main/resources/dynamodb.md +++ b/adapter-dynamodb/src/main/resources/dynamodb.md @@ -1,4 +1,4 @@ -# dynamodb driver +# dynamodb The DynamoDB driver supports a basic set of commands as specified at [Amazon DynamoDB Docs](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Operations_Amazon_DynamoDB.html). diff --git a/adapter-http/src/main/resources/http.md b/adapter-http/src/main/resources/http.md index d2ec56512..545ce7080 100644 --- a/adapter-http/src/main/resources/http.md +++ b/adapter-http/src/main/resources/http.md @@ -1,4 +1,4 @@ -# HTTP driver +# http This driver allows you to make http requests using the native HTTP client that is bundled with the JVM. It supports free-form construction of @@ -23,8 +23,8 @@ Or, you can have a list: ```yaml # A list of statements ops: - - http://google.com/ - - http://amazon.com/ + - http://google.com/ + - http://amazon.com/ ``` Or you can template the values used in the URI, and even add ratios: diff --git a/adapter-mongodb/src/main/resources/mongodb.md b/adapter-mongodb/src/main/resources/mongodb.md index 51df63ddd..b60008a04 100644 --- a/adapter-mongodb/src/main/resources/mongodb.md +++ b/adapter-mongodb/src/main/resources/mongodb.md @@ -1,4 +1,4 @@ -# MongoDB Driver +# MongoDB This is a driver for MongoDB. It supports the `db.runCommand` API described in [here](https://docs.mongodb.com/manual/reference/command/). @@ -10,7 +10,9 @@ for details on command structure. Run a mongodb activity with definitions from activities/mongodb-basic.yaml - ... driver=mongodb yaml=activities/mongo-basic.yaml +```shell +nb5 driver=mongodb yaml=activities/mongo-basic.yaml +``` ### MongoDB ActivityType Parameters diff --git a/adapter-pulsar/src/main/resources/pulsar.md b/adapter-pulsar/src/main/resources/pulsar.md index 9ac6e4ae2..92ad5a189 100644 --- a/adapter-pulsar/src/main/resources/pulsar.md +++ b/adapter-pulsar/src/main/resources/pulsar.md @@ -1,15 +1,4 @@ -# Table of contents -- [1. Overview](#1-overview) - - [1.1. Issues Tracker](#1-1-issues-tracker) -- [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload) - - [2.1. NB Pulsar Driver Yaml File High Level Structure](#2-1-nb-pulsar-driver-yaml-file-high-level-structure) - - [2.2. NB Pulsar Driver Configuration Parameters](#2-2-nb-pulsar-driver-configuration-parameters) - - [2.2.1. Global Level Parameters](#2-2-1-global-level-parameters) - - [2.2.2. Document Level Parameters](#2-2-2-document-level-parameters) -- [3. NB Pulsar Driver OpTemplates](#3-nb-pulsar-driver-optemplates) -- [4. Message Generation and Schema Support](#4-message-generation-and-schema-support) - - [4.1. Message Generation](#4-1-message-generation) - - [4.2. Schema Support](#4-2-schema-support) +# pulsar # 1. Overview diff --git a/adapter-s4j/src/main/resources/s4j.md b/adapter-s4j/src/main/resources/s4j.md index c24b35d9a..0fb58ccce 100644 --- a/adapter-s4j/src/main/resources/s4j.md +++ b/adapter-s4j/src/main/resources/s4j.md @@ -1,18 +1,7 @@ --- weight: 0 -title: S4J Adapter +title: S4J --- -# S4J Adapter -- [1. Overview](#1-overview) -- [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload) -- [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file) -- [4. NB S4J Scenario Definition File](#4-nb-s4j-scenario-definition-file) - - [4.1. Document Level Parameters](#4-1-document-level-parameters) - - [4.2. NB S4J Workload Types](#4-2-nb-s4j-workload-types) - - [4.2.1. Publish Messages to a JMS Destination, Queue or Topic](#4-2-1-publish-messages-to-a-jms-destination-queue-or-topic) - - [4.2.2. Receiving Messages from a JMS Destination, Queue or Topic](#4-2-2-receiving-messages-from-a-jms-destination-queue-or-topic) - - # 1. Overview This driver is similar to [NB Pulsar driver](../../../../driver-pulsar/src/main/resources/pulsar.md) that allows NB based workload generation and performance testing against a Pulsar cluster. It also follows a similar pattern to configure and connect to the Pulsar cluster for workload execution. @@ -23,7 +12,7 @@ However, the major difference is instead of simulating native Pulsar client work The following is an example of executing a NB S4J workload (defined as *pulsar_s4j.yaml*) -``` +```shell $ run driver=s4j cycles=10000 threads=4 num_conn=2 num_session=2 session_mode="client_ack" strict_msg_error_handling="false" web_url=http://localhost:8080 service_url=pulsar://localhost:6650 config=/path/to/nb_s4j_config.properties yaml=/path/to/pulsar_s4j.yaml -vv --logs-dir=s4j_log ``` @@ -153,7 +142,7 @@ The NB S4J statement block for publishing messages to a JMS destination (either * The default message type (**msg_type**) is "byte". But optionally, you can specify other message types such as "text", "map", etc. * The message payload (**msg_body**) is the only mandatory field. -``` +```yaml blocks: msg-produce-block: ops: @@ -196,7 +185,8 @@ The generic NB S4J statement block for receiving messages to a JMS destination ( * negative ack/ack timeout/deadletter topic related settings * The settings here (as the scenario specific settings) will be merged with the * global settings in *s4j_config.properties* file -``` + +```yaml blocks: msg-produce-block: ops: diff --git a/adapter-stdout/src/main/resources/stdout.md b/adapter-stdout/src/main/resources/stdout.md index 9feb8f43f..2490f3c25 100644 --- a/adapter-stdout/src/main/resources/stdout.md +++ b/adapter-stdout/src/main/resources/stdout.md @@ -1,4 +1,4 @@ -# stdout activity type +# stdout This is an activity type which allows for the generation of data into to stdout or a file. It reads the standard nosqlbench YAML @@ -9,15 +9,21 @@ that uses the curly brace token form in statements. Run a stdout activity named 'stdout-test', with definitions from activities/stdout-test.yaml - ... driver=stdout workload=stdout-test +```shell +nb5 driver=stdout workload=stdout-test +``` Only run statement groups which match a tag regex - ... driver=stdout workload=stdout-test tags=group:'ddl.*' +```shell +nb5 driver=stdout workload=stdout-test tags=group:'ddl.*' +``` Run the matching 'dml' statements, with 100 cycles, from [1000..1100) - ... driver=stdout workload=stdout-test tags=group:'dml.*' cycles=1000..11000 filename=test.csv +```shell +nb5 driver=stdout workload=stdout-test tags=group:'dml.*' cycles=1000..11000 filename=test.csv +``` This last example shows that the cycle range is [inclusive..exclusive), to allow for stacking test intervals. This is standard across all @@ -60,15 +66,21 @@ For more details on this format, please refer to the The statement format for this activity type is a simple string. Tokens between curly braces are used to refer to binding names, as in the following example: - statements: - - "It is {minutes} past {hour}." +```yaml + ops: + op1: "It is {minutes} past {hour}." +``` If you want to suppress the trailing newline that is automatically added, then you must either pass `newline=false` as an activity param, or specify it in the statement params in your config as in: - params: - newline: false +```yaml +ops: + op1: + stmt: "It is {minutes} past {hour}." + newline: false +``` ### Auto-generated statements @@ -76,13 +88,17 @@ If no statement is provided, then the defined binding names are used as-is to create a CSV-style line format. The values are concatenated with comma delimiters, so a set of bindings like this: +```yaml bindings: one: Identity() two: NumberNameToString() +``` would create an automatic string template like this: - statements: - - "{one},{two}\n" +```yaml +ops: + op1: "{one},{two}\n" +``` The auto-generation behavior is forced when the format parameter is supplied. diff --git a/adapter-tcp/src/main/resources/tcpclient.md b/adapter-tcp/src/main/resources/tcpclient.md index f0eb7777b..641b66ca3 100644 --- a/adapter-tcp/src/main/resources/tcpclient.md +++ b/adapter-tcp/src/main/resources/tcpclient.md @@ -1,4 +1,4 @@ -# Driver: tcpclient +# tcpclient **tcpclient acts like a _client push_ version of stdout over TCP** diff --git a/adapter-tcp/src/main/resources/tcpserver.md b/adapter-tcp/src/main/resources/tcpserver.md index 80699599d..73a4c0222 100644 --- a/adapter-tcp/src/main/resources/tcpserver.md +++ b/adapter-tcp/src/main/resources/tcpserver.md @@ -1,4 +1,4 @@ -# Driver: tcpserver +# tcpserver **tcpserver acts like a _server push_ version of stdout over TCP** diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/docs/BundledDriverAdapterDocs.java similarity index 88% rename from adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java rename to adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/docs/BundledDriverAdapterDocs.java index d1f69307d..ce18e7294 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/docs/BundledDriverAdapterDocs.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,20 +14,17 @@ * limitations under the License. */ -package io.nosqlbench.engine.api.activityimpl.uniform; +package io.nosqlbench.engine.api.activityimpl.docs; import io.nosqlbench.api.docsapi.BundledMarkdownManifest; import io.nosqlbench.api.docsapi.Docs; import io.nosqlbench.api.docsapi.DocsBinder; -import io.nosqlbench.api.docsapi.DocsNameSpace; +import io.nosqlbench.api.spi.SimpleServiceLoader; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.nb.annotations.Maturity; import io.nosqlbench.nb.annotations.Service; -import io.nosqlbench.api.spi.SimpleServiceLoader; -import java.nio.file.Path; import java.util.List; -import java.util.Map; -import java.util.Set; @Service(value = BundledMarkdownManifest.class, selector = "drivers") public class BundledDriverAdapterDocs implements BundledMarkdownManifest { diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java new file mode 100644 index 000000000..df2cf2be3 --- /dev/null +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.engine.api.specifications; + +import io.nosqlbench.api.content.Content; +import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.api.docsapi.BundledMarkdownManifest; +import io.nosqlbench.api.docsapi.Docs; +import io.nosqlbench.api.docsapi.DocsBinder; +import io.nosqlbench.nb.annotations.Service; + +import java.util.List; + +@Service(value = BundledMarkdownManifest.class, selector = "specifications") +public class SpecDocsManifest implements BundledMarkdownManifest { + @Override + public DocsBinder getDocs() { + Docs docs = new Docs().namespace("workload_definition"); + List> specfiles = NBIO.classpath().prefix("workload_definition/").extension(".md").list(); + for (Content specfile : specfiles) { + docs.addPath(specfile.asPath()); + } + return docs; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/api/spi/BundledApp.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/SpecificationDocsManifest.java similarity index 57% rename from nb-api/src/main/java/io/nosqlbench/api/spi/BundledApp.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/SpecificationDocsManifest.java index 50273225d..c2c901a64 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/spi/BundledApp.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/SpecificationDocsManifest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,17 @@ * limitations under the License. */ -package io.nosqlbench.api.spi; +package io.nosqlbench.engine.api.activityconfig.specifications; -import java.util.function.ToIntFunction; +import io.nosqlbench.api.docsapi.BundledMarkdownManifest; +import io.nosqlbench.api.docsapi.Docs; +import io.nosqlbench.api.docsapi.DocsBinder; -public interface BundledApp extends ToIntFunction { +public class SpecificationDocsManifest implements BundledMarkdownManifest { + @Override + public DocsBinder getDocs() { + Docs docs = new Docs(); + return docs; + } } diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/UniformWorkloadSpecificationTest.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/UniformWorkloadSpecificationTest.java similarity index 94% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/UniformWorkloadSpecificationTest.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/UniformWorkloadSpecificationTest.java index aa77bb392..dab2a6b12 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/UniformWorkloadSpecificationTest.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/UniformWorkloadSpecificationTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.engine.api.activityconfig.rawyaml; +package io.nosqlbench.engine.api.activityconfig.specifications; import com.vladsch.flexmark.ast.FencedCodeBlock; import io.nosqlbench.nb.spectest.core.SpecTest; diff --git a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/YamlSpecValidator.java b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java similarity index 97% rename from adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/YamlSpecValidator.java rename to adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java index ad5e99039..86e307e44 100644 --- a/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/YamlSpecValidator.java +++ b/adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,12 @@ * limitations under the License. */ -package io.nosqlbench.engine.api.activityconfig.rawyaml; +package io.nosqlbench.engine.api.activityconfig.specifications; import com.google.gson.*; import com.google.gson.reflect.TypeToken; import io.nosqlbench.engine.api.activityconfig.StatementsLoader; +import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader; import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.nb.spectest.api.STAssemblyValidator; diff --git a/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServerApp.java b/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServerApp.java index a91978130..f9d54b090 100644 --- a/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServerApp.java +++ b/docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServerApp.java @@ -16,7 +16,7 @@ package io.nosqlbench.docsys.core; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.nb.annotations.Service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index e768f6cb0..ef0878db3 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -25,7 +25,7 @@ import io.nosqlbench.api.errors.BasicError; import io.nosqlbench.api.logging.NBLogLevel; import io.nosqlbench.api.metadata.SessionNamer; import io.nosqlbench.api.metadata.SystemId; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility; import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility; import io.nosqlbench.engine.api.activityapi.input.InputType; diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 78edba3ee..2751f5d60 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -445,6 +445,16 @@ src/test/resources + + log4j2-test.xml + + true + + + src/test/resources + + log4j2-test.xml + true diff --git a/nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java b/nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java new file mode 100644 index 000000000..29a798ddd --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022-2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.api.apps; + +import io.nosqlbench.api.content.Content; +import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.api.docsapi.Docs; +import io.nosqlbench.api.docsapi.DocsBinder; +import io.nosqlbench.nb.annotations.Service; + +import java.util.Optional; +import java.util.function.ToIntFunction; + +public interface BundledApp extends ToIntFunction { + + int applyAsInt(String[] value); + + default String getBundledAppName() { + return this.getClass().getAnnotation(Service.class).selector(); + } + + default DocsBinder getBundledDocs() { + Docs docs = new Docs().namespace("apps"); + + String dev_docspath = "app-" + this.getBundledAppName() + "/src/main/resources/docs/" + this.getBundledAppName(); + String cp_docspath = "docs/" + this.getBundledAppName(); + Optional> bundled_docs = NBIO.local().name(dev_docspath, cp_docspath).first(); + bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf); + + Optional> maindoc = NBIO.local().name("/src/main/resources/" + this.getBundledAppName() + ".md", this.getBundledAppName() + ".md").first(); + + maindoc.map(Content::asPath).ifPresent(docs::addPath); + + return docs.asDocsBinder(); + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/api/apps/BundledAppDocs.java b/nb-api/src/main/java/io/nosqlbench/api/apps/BundledAppDocs.java new file mode 100644 index 000000000..b4ac7daf7 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/api/apps/BundledAppDocs.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.api.apps; + +import io.nosqlbench.api.docsapi.BundledMarkdownManifest; +import io.nosqlbench.api.docsapi.Docs; +import io.nosqlbench.api.docsapi.DocsBinder; +import io.nosqlbench.api.spi.SimpleServiceLoader; +import io.nosqlbench.nb.annotations.Maturity; +import io.nosqlbench.nb.annotations.Service; + +import java.util.List; + +@Service(value = BundledMarkdownManifest.class, selector = "apps") +public class BundledAppDocs implements BundledMarkdownManifest { + @Override + public DocsBinder getDocs() { + DocsBinder docs = new Docs(); + SimpleServiceLoader loader = new SimpleServiceLoader<>(BundledApp.class, Maturity.Any); + List> namedProviders = loader.getNamedProviders(); + for (SimpleServiceLoader.Component namedProvider : namedProviders) { + BundledApp app = namedProvider.provider.get(); + DocsBinder bundledDocs = app.getBundledDocs(); + docs = docs.merge(bundledDocs); + } + return docs; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java b/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java index 050c51c57..3cba8a068 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java +++ b/nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java @@ -18,6 +18,7 @@ package io.nosqlbench.api.docsapi; import java.util.ServiceLoader; + public class BundledMarkdownLoader { public static DocsBinder loadBundledMarkdown() { diff --git a/nb-api/src/main/java/io/nosqlbench/api/markdown/exporter/MarkdownExporter.java b/nb-api/src/main/java/io/nosqlbench/api/markdown/exporter/MarkdownExporter.java index 2a57a7a0e..3972457b7 100644 --- a/nb-api/src/main/java/io/nosqlbench/api/markdown/exporter/MarkdownExporter.java +++ b/nb-api/src/main/java/io/nosqlbench/api/markdown/exporter/MarkdownExporter.java @@ -19,7 +19,7 @@ package io.nosqlbench.api.markdown.exporter; import io.nosqlbench.api.markdown.aggregator.MarkdownDocs; import io.nosqlbench.api.markdown.types.DocScope; import io.nosqlbench.api.markdown.types.MarkdownInfo; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.nb.annotations.Service; import joptsimple.*; diff --git a/nb-spectest/src/main/java/io/nosqlbench/nb/spectest/core/SpecTest.java b/nb-spectest/src/main/java/io/nosqlbench/nb/spectest/core/SpecTest.java index 6d5bb76c3..b7437e221 100644 --- a/nb-spectest/src/main/java/io/nosqlbench/nb/spectest/core/SpecTest.java +++ b/nb-spectest/src/main/java/io/nosqlbench/nb/spectest/core/SpecTest.java @@ -24,6 +24,7 @@ import io.nosqlbench.nb.spectest.api.STBuilderFacets; import io.nosqlbench.nb.spectest.api.STPathLoader; import java.nio.file.Path; +import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -86,6 +87,15 @@ public class SpecTest implements Runnable { } } + public List matchingSpecFiles() { + List paths = new ArrayList<>(); + for (Path path : paths) { + List matchingPaths = STFileScanner.findMatching(".*\\.md", paths.toArray(new Path[0])); + paths.addAll(matchingPaths); + } + return paths; + } + @Override public void run() { Set testables = new LinkedHashSet<>(); diff --git a/nbr/pom.xml b/nbr/pom.xml index e58823eaa..044e1210d 100644 --- a/nbr/pom.xml +++ b/nbr/pom.xml @@ -73,6 +73,10 @@ src/main/resources + + version.properties + nb_version_info.md + true diff --git a/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java index a6c5ecc07..62a4a1e26 100644 --- a/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java +++ b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java @@ -16,7 +16,7 @@ package io.nosqlbench.api.docsapi.docexporter; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.nb.annotations.Service; import joptsimple.OptionParser; import joptsimple.OptionSet; diff --git a/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java new file mode 100644 index 000000000..6c720339b --- /dev/null +++ b/nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.api.docsapi.docexporter; + +import io.nosqlbench.api.content.NBIO; +import io.nosqlbench.api.docsapi.BundledMarkdownManifest; +import io.nosqlbench.api.docsapi.Docs; +import io.nosqlbench.api.docsapi.DocsBinder; +import io.nosqlbench.nb.annotations.Service; + +@Service(value=BundledMarkdownManifest.class,selector = "versions") +public class BundledVersionManifest implements BundledMarkdownManifest { + @Override + public DocsBinder getDocs() { + return new Docs() + .namespace("versions") + .addPath(NBIO.classpath().name("nb_version_info.md").one().asPath()) + .asDocsBinder(); + } +} diff --git a/nbr/src/main/resources/nb_version_info.md b/nbr/src/main/resources/nb_version_info.md new file mode 100644 index 000000000..f041ef7e7 --- /dev/null +++ b/nbr/src/main/resources/nb_version_info.md @@ -0,0 +1,9 @@ +# Version Info + +The version of NB5 which provided some of the docs content is: + +``` +version=${project.version} +groupId=${project.groupId} +artifactId=${project.artifactId} +``` diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java index d34fe2c0c..0859dc53f 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java @@ -16,10 +16,9 @@ package io.nosqlbench.virtdata.userlibs.apps; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.virtdata.userlibs.apps.diagnoseapp.VirtDataDiagnoseApp; -import io.nosqlbench.virtdata.userlibs.apps.docsapp.VirtDataGenDocsApp; import io.nosqlbench.virtdata.userlibs.apps.valuechecker.VirtDataCheckPerfApp; import java.util.Arrays; diff --git a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/summarizer/ValueSummarizerApp.java b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/summarizer/ValueSummarizerApp.java index 74193a290..aa59b45b4 100644 --- a/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/summarizer/ValueSummarizerApp.java +++ b/virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/summarizer/ValueSummarizerApp.java @@ -16,7 +16,7 @@ package io.nosqlbench.virtdata.userlibs.apps.summarizer; -import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.api.apps.BundledApp; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.virtdata.core.bindings.DataMapper; import io.nosqlbench.virtdata.core.bindings.VirtData;