diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f9e0bb6b1..2d37826ef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,51 +81,27 @@ jobs: if: ${{ github.repository == 'nosqlbench/nosqlbench' && github.event_name == 'pull_request' && github.ref_name == 'main' }} steps: - - name: set git username - run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}" - - - name: set git email - run: git config --global user.name "${{ secrets.NBDROID_NAME }}" + - name: checkout build docs + uses: actions/checkout@v4 + with: + repository: nosqlbench/nosqlbench-build-docs + path: build-docs - name: download exported_docs uses: actions/download-artifact@v4 with: name: exported-docs - - run: ls -la - - - name: clone nosqlbench-build-docs - env: - NBDROID_NAME: ${{ secrets.NBDROID_NAME }} - NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }} - run: | - git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs - cd nosqlbench-build-docs - echo "files listing" - find . - git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git - git remote -v - - - name: import zipfile - run: | - mv exported_docs.zip nosqlbench-build-docs/ - - - name: set CNAME - run: | - echo "builddocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME - - - name: commit changes - run: | - cd nosqlbench-build-docs - git add exported_docs.zip - - - name: push changes - env: - NBDROID_NAME: ${{ secrets.NBDROID_NAME }} - NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }} + - name: overlay docs changes and push run: | set -x - cd nosqlbench-build-docs + mv exported_docs.zip build-docs/. + cd build-docs + ./doimport.sh + rm exported_docs.zip + git config --global user.email "${{ secrets.NBDROID_EMAIL }}" + git config --global user.name "${{ secrets.NBDROID_NAME }}" + git add . CHANGES=$(git status --porcelain 2>/dev/null| wc -l) echo "found $CHANGES to push for doc updates" if (( $CHANGES > 0 )) @@ -133,4 +109,3 @@ jobs: git commit -m"docs update for $GITHUB_REF" git push fi - echo "push completed" diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index f5280aab9..3c6355c1c 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -698,7 +698,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.2.1 + 3.5.0 enforce-java @@ -716,6 +716,19 @@ + + + org.cyclonedx + cyclonedx-maven-plugin + + + install + + makeAggregateBom + + + + @@ -723,18 +736,17 @@ org.apache.maven.plugins maven-assembly-plugin - 3.4.2 + 3.7.1 - org.apache.maven.plugins maven-release-plugin - 3.0.0-M7 + 3.1.1 org.apache.maven.plugins maven-gpg-plugin - 3.0.1 + 3.2.7 org.apache.maven.plugins @@ -744,53 +756,53 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M8 + 3.5.2 org.apache.maven.plugins maven-failsafe-plugin - 3.0.0-M8 + 3.5.2 org.apache.maven.plugins maven-javadoc-plugin - 3.4.1 + 3.11.2 org.apache.maven.plugins maven-source-plugin - 3.2.1 + 3.3.1 org.sonatype.plugins nexus-staging-maven-plugin - 1.6.13 + 1.7.0 org.antlr antlr4-maven-plugin - 4.13.1 + 4.13.2 org.codehaus.mojo exec-maven-plugin - 3.1.0 + 3.5.0 org.apache.maven.plugins maven-enforcer-plugin - 3.2.1 + 3.5.0 org.apache.maven.plugins maven-clean-plugin - 3.2.0 + 3.4.0 org.apache.maven.plugins maven-resources-plugin - 3.3.0 + 3.3.1 org.jacoco @@ -800,17 +812,17 @@ org.apache.maven.plugins maven-jar-plugin - 3.3.0 + 3.4.2 org.apache.maven.plugins maven-install-plugin - 3.1.0 + 3.1.3 org.apache.maven.plugins maven-deploy-plugin - 3.0.0 + 3.1.3 @@ -958,7 +970,7 @@ attach-javadoc - aggregate-jar + aggregate diff --git a/nb-adapters/adapter-amqp/src/main/resources/amqp.md b/nb-adapters/adapter-amqp/src/main/resources/amqp.md index bf20cc0f4..a5bb1366c 100644 --- a/nb-adapters/adapter-amqp/src/main/resources/amqp.md +++ b/nb-adapters/adapter-amqp/src/main/resources/amqp.md @@ -1,6 +1,7 @@ --- weight: 0 title: AMQP +source: nb-adapters/adapter-amqp/src/main/resources/amqp.md --- - [1. Overview](#1-overview) - [2. NB AMQP Usage](#2-nb-amqp-usage) diff --git a/nb-adapters/adapter-cqld4/src/main/resources/cqld4.md b/nb-adapters/adapter-cqld4/src/main/resources/cqld4.md index fde0a2a6f..44315f53d 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/cqld4.md +++ b/nb-adapters/adapter-cqld4/src/main/resources/cqld4.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-cqld4/src/main/resources/cqld4.md +--- # cqld4 This is the newly revamped driver for CQL which uses the DataStax OSS Driver version 4. As diff --git a/nb-adapters/adapter-cqld4/src/main/resources/cqlgen.md b/nb-adapters/adapter-cqld4/src/main/resources/cqlgen.md index 083648194..ad4fa5162 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/cqlgen.md +++ b/nb-adapters/adapter-cqld4/src/main/resources/cqlgen.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-cqld4/src/main/resources/cqlgen.md +--- # cqlgen *cqlgen* is a default CQL workload generator diff --git a/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md b/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md index ff06ff56b..7c7f9f02e 100644 --- a/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md +++ b/nb-adapters/adapter-dataapi/src/main/resources/dataapi.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-dataapi/src/main/resources/dataapi.md +--- # Data API ## DataStax Data API Adapter diff --git a/nb-adapters/adapter-diag/src/main/resources/diag.md b/nb-adapters/adapter-diag/src/main/resources/diag.md index 5d5b7705d..941169918 100644 --- a/nb-adapters/adapter-diag/src/main/resources/diag.md +++ b/nb-adapters/adapter-diag/src/main/resources/diag.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-diag/src/main/resources/diag.md +--- # diag This is a diagnostic activity type. Its action simply reports the cycle number and the reporting delay between it's schedule reporting time and the current time. The reporting is interleaved between the threads, with the logical number of reports remaining constant regardless of the thread count. diff --git a/nb-adapters/adapter-dynamodb/src/main/resources/dynamodb.md b/nb-adapters/adapter-dynamodb/src/main/resources/dynamodb.md index 90969c2cd..4c9a84a18 100644 --- a/nb-adapters/adapter-dynamodb/src/main/resources/dynamodb.md +++ b/nb-adapters/adapter-dynamodb/src/main/resources/dynamodb.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-dynamodb/src/main/resources/dynamodb.md +--- # dynamodb The DynamoDB driver supports a basic set of commands as specified at diff --git a/nb-adapters/adapter-example/src/main/resources/example.md b/nb-adapters/adapter-example/src/main/resources/example.md new file mode 100644 index 000000000..eaf3ec67a --- /dev/null +++ b/nb-adapters/adapter-example/src/main/resources/example.md @@ -0,0 +1,6 @@ +--- +source: nb-adapters/adapter-example/src/main/resources/example.md +--- +# Example + +Stub. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java index b68fcf0fb..ef7f926a6 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java @@ -60,7 +60,7 @@ public class GCPSpannerInsertOpDispenser extends GCPSpannerBaseOpDispenser createParamsFunction(ParsedOp op) { return op.getAsOptionalFunction("query_params", Map.class) - .orElse(_ -> Collections.emptyMap()); + .orElse(l -> Collections.emptyMap()); } /** diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md b/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md index 5642e2323..b0011080c 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md +--- # Google Spanner driver adapter The Google Cloud Spanner driver adapter is a NoSQLBench adapter for the `gcp_spanner` driver, a Java driver for connecting to and performing operations on an instance of a Google Cloud Spanner database. diff --git a/nb-adapters/adapter-http/src/main/resources/http.md b/nb-adapters/adapter-http/src/main/resources/http.md index 8664a443d..1b7388b98 100644 --- a/nb-adapters/adapter-http/src/main/resources/http.md +++ b/nb-adapters/adapter-http/src/main/resources/http.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-http/src/main/resources/http.md +--- # http This driver allows you to make http requests using the native HTTP client diff --git a/nb-adapters/adapter-kafka/src/main/resources/kafka.md b/nb-adapters/adapter-kafka/src/main/resources/kafka.md index 6fb05d4cb..e8e0f3f5b 100644 --- a/nb-adapters/adapter-kafka/src/main/resources/kafka.md +++ b/nb-adapters/adapter-kafka/src/main/resources/kafka.md @@ -1,6 +1,7 @@ --- weight: 0 title: Kafka +source: nb-adapters/adapter-kafka/src/main/resources/kafka.md --- - [1. Overview](#1-overview) - [1.1. Example NB Yaml](#11-example-nb-yaml) diff --git a/nb-adapters/adapter-mongodb/src/main/resources/mongodb.md b/nb-adapters/adapter-mongodb/src/main/resources/mongodb.md index b60008a04..a6de1d90b 100644 --- a/nb-adapters/adapter-mongodb/src/main/resources/mongodb.md +++ b/nb-adapters/adapter-mongodb/src/main/resources/mongodb.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-mongodb/src/main/resources/mongodb.md +--- # MongoDB This is a driver for MongoDB. It supports the `db.runCommand` API described in [here](https://docs.mongodb.com/manual/reference/command/). diff --git a/nb-adapters/adapter-neo4j/src/main/resources/neo4j.md b/nb-adapters/adapter-neo4j/src/main/resources/neo4j.md index e986529d0..6af2f3bb0 100644 --- a/nb-adapters/adapter-neo4j/src/main/resources/neo4j.md +++ b/nb-adapters/adapter-neo4j/src/main/resources/neo4j.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-neo4j/src/main/resources/neo4j.md +--- # neo4j driver adapter The neo4j driver adapter is a nb adapter for the Neo4J driver, an open source Java driver for connecting to and diff --git a/nb-adapters/adapter-pinecone/src/main/resources/pinecone.md b/nb-adapters/adapter-pinecone/src/main/resources/pinecone.md index 766516c97..3c550806b 100644 --- a/nb-adapters/adapter-pinecone/src/main/resources/pinecone.md +++ b/nb-adapters/adapter-pinecone/src/main/resources/pinecone.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-pinecone/src/main/resources/pinecone.md +--- # pinecone driver adapter The pinecone driver adapter is a nb adapter for the pinecone driver, an open source Java driver for connecting to and performing operations on an instance of a Pinecone Vector database. The driver is hosted on github at diff --git a/nb-adapters/adapter-pulsar/src/main/resources/pulsar.md b/nb-adapters/adapter-pulsar/src/main/resources/pulsar.md index 55b989178..47681ffc1 100644 --- a/nb-adapters/adapter-pulsar/src/main/resources/pulsar.md +++ b/nb-adapters/adapter-pulsar/src/main/resources/pulsar.md @@ -1,8 +1,10 @@ --- weight: 0 -title: Pulsar +source: nb-adapters/adapter-pulsar/src/main/resources/pulsar.md --- +# Pulsar + - [1. Overview](#1-overview) - [1.1. Issues Tracker](#11-issues-tracker) - [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload) diff --git a/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md b/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md index 711700a89..b6510b35e 100644 --- a/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md +++ b/nb-adapters/adapter-qdrant/src/main/resources/qdrant.md @@ -1,3 +1,7 @@ +--- +source: nb-adapters/adapter-qdrant/src/main/resources/qdrant.md +--- + # qdrant driver adapter The qdrant driver adapter is a nb adapter for the qdrant driver, an open source Java driver for connecting to and diff --git a/nb-adapters/adapter-s4j/src/main/resources/s4j.md b/nb-adapters/adapter-s4j/src/main/resources/s4j.md index f85cb8146..6272d2909 100644 --- a/nb-adapters/adapter-s4j/src/main/resources/s4j.md +++ b/nb-adapters/adapter-s4j/src/main/resources/s4j.md @@ -1,6 +1,7 @@ --- weight: 0 title: S4J +source: nb-adapters/adapter-s4j/src/main/resources/s4j.md --- - [1. Overview](#1-overview) - [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload) diff --git a/nb-adapters/adapter-stdout/src/main/resources/stdout.md b/nb-adapters/adapter-stdout/src/main/resources/stdout.md index 2490f3c25..b56d20e82 100644 --- a/nb-adapters/adapter-stdout/src/main/resources/stdout.md +++ b/nb-adapters/adapter-stdout/src/main/resources/stdout.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-stdout/src/main/resources/stdout.md +--- # stdout This is an activity type which allows for the generation of data diff --git a/nb-adapters/adapter-tcp/src/main/resources/tcpclient.md b/nb-adapters/adapter-tcp/src/main/resources/tcpclient.md index a2dbbbc60..2f5ac2d38 100644 --- a/nb-adapters/adapter-tcp/src/main/resources/tcpclient.md +++ b/nb-adapters/adapter-tcp/src/main/resources/tcpclient.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-tcp/src/main/resources/tcpclient.md +--- # tcpclient **tcpclient acts like a _client push_ version of stdout over TCP** diff --git a/nb-adapters/adapter-tcp/src/main/resources/tcpserver.md b/nb-adapters/adapter-tcp/src/main/resources/tcpserver.md index 7b77dda78..eaf7b0ebf 100644 --- a/nb-adapters/adapter-tcp/src/main/resources/tcpserver.md +++ b/nb-adapters/adapter-tcp/src/main/resources/tcpserver.md @@ -1,3 +1,6 @@ +--- +source: nb-adapters/adapter-tcp/src/main/resources/tcpserver.md +--- # tcpserver **tcpserver acts like a _server push_ version of stdout over TCP** diff --git a/nb-adapters/adapter-weaviate/src/main/resources/weaviate.md b/nb-adapters/adapter-weaviate/src/main/resources/weaviate.md index 76f73c027..fa1a095a1 100644 --- a/nb-adapters/adapter-weaviate/src/main/resources/weaviate.md +++ b/nb-adapters/adapter-weaviate/src/main/resources/weaviate.md @@ -1,7 +1,10 @@ +--- +source: nb-adapters/adapter-weaviate/src/main/resources/weaviate.md +--- # Weaviate driver adapter The Weaviate driver adapter is a NoSQLBench adapter for the `weaviate` driver, an open-source Java driver -for connecting to and performing operations on an instance of a Weaviate vector database. The driver is +for connecting to and performing operations on an instance of a Weaviate vector database. The driver is leveraged from GitHub at https://github.com/weaviate/java-client. ## Run Commands (Remove prior to merge) @@ -39,8 +42,8 @@ The following parameters must be supplied to the adapter at runtime in order to instance of the [Weaviate database](https://weaviate.io/developers/weaviate): * `token` - In order to use the Weaviate database you must have an account. Once the account is created you can [request - an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be - provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and + an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be + provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and referenced via the `token_file` config option pointing to the path of the file. * `uri` - When a collection/index is created in the database the URI (aka endpoint) must be specified as well. The adapter will use the default value of `localhost:8080` if none is provided at runtime. Remember to *not* provide the `https://` diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 68b3b5f82..99daafb54 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -186,7 +186,7 @@ adapter-pulsar-include - false + true diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java index b694e4c73..e0cb9a50b 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/DriverAdapter.java @@ -269,10 +269,14 @@ public interface DriverAdapter, SPACETYPE extends Spac Optional> bundled_docs = NBIO.local().pathname( dev_docspath, cp_docspath).first(); bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf); + String markdown = this.getAdapterName() +".md"; + System.out.println("Markdown: " + markdown); Optional> maindoc = NBIO.local().pathname( - "/src/main/resources/" + this.getAdapterName() + ".md", - this.getAdapterName() + ".md" + "/src/main/resources/" + markdown, markdown ).first(); + if ( !maindoc.isPresent() ) { + System.out.println("Could not find main document for " + markdown); + } maindoc.map(Content::asPath).ifPresent(docs::addPath); return docs.asDocsBinder(); } diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md b/nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md index 90a1c2761..dc417876d 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md +--- # Workload Specification This directory contains the testable specification for workload definitions used by NoSQLBench. diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md b/nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md index da4677746..70a6d3c29 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md +--- # SpecTest Formatting The specifications and examples follow a pattern: diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md b/nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md index ee6cc2792..b5b8f38e9 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md +--- # Workload Structure ## Keywords diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md b/nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md index c65ad42e1..627cc646b 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md +--- # Op Templates Op templates are the recipes provided by users for an operation. These hold examples of payload diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md b/nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md index 402edcbb0..84407d3ac 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md +--- # Op Template Payloads Payloads in NoSQLBench op templates can be of any type that you can create from bindings, string diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md b/nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md index e01e37bf4..61e8f46d0 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md @@ -1,4 +1,5 @@ --- +source: nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md title: Op Template Variations --- diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md b/nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md index 874389de2..df5744daa 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md +--- # Template Variables Template variables allow for workload descriptions to be parameterized outside the structure of the diff --git a/nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md b/nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md index 8d0686fb2..5bad6633e 100644 --- a/nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md +++ b/nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md @@ -1,3 +1,6 @@ +--- +source: nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md +--- # ParsedOp API In the workload template examples, we show statements as being formed from a string value. This is a diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java index 037b43507..c6f57a394 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java @@ -28,6 +28,7 @@ import java.util.Optional; public class MutableFrontMatter extends LinkedHashMap> { String WEIGHT = "weight"; String TITLE = "title"; + String SOURCE = "source"; MutableFrontMatter(Map> data) { this.putAll(data); @@ -43,6 +44,11 @@ public class MutableFrontMatter extends LinkedHashMap> { return Optional.ofNullable(get(WEIGHT)).map(l -> l.get(0)).map(Integer::parseInt).orElse(0); } + public String getSource() { + assertMaxSingleValued(SOURCE); + return Optional.ofNullable(get(SOURCE)).map(l -> l.get(0)).orElse(null); + } + public void setTitle(String title) { put(TITLE,List.of(title)); } @@ -51,6 +57,10 @@ public class MutableFrontMatter extends LinkedHashMap> { put(WEIGHT,List.of(String.valueOf(weight))); } + public void setSource(String source) { + put(SOURCE,List.of(source)); + } + private void assertMaxSingleValued(String fieldname) { if (containsKey(fieldname) && get(fieldname).size()>1) { throw new RuntimeException("Field '" + fieldname + "' can only have zero or one value. It is single-valued."); @@ -60,7 +70,8 @@ public class MutableFrontMatter extends LinkedHashMap> { public String asYaml() { DumpSettings settings = DumpSettings.builder().setDefaultFlowStyle(FlowStyle.BLOCK).build(); Dump dump = new Dump(settings); - return dump.dumpToString(Map.of(TITLE,getTitle(),WEIGHT,getWeight())); + return dump.dumpToString(Map.of(TITLE,getTitle(),WEIGHT,getWeight(),SOURCE,getSource())); } + } diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java index dcb4a8228..a817ba6aa 100644 --- a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdown.java @@ -63,7 +63,16 @@ public class MutableMarkdown { v.visit(parsed); Map> data = v.getData(); this.frontMatter = new MutableFrontMatter(data); - + if(data.containsKey("title")) { + this.frontMatter.setTitle(data.get("title").getFirst().toString()); + } + if(data.containsKey("source")) { + this.frontMatter.setSource("https://github.com/nosqlbench/nosqlbench/blob/main/"+data.get("source").getFirst().toString()); + } else if (this.path != null) { + this.frontMatter.setSource(this.path.getFileName().toString()); + } else { + this.frontMatter.setSource("generated by export-docs"); + } if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) { Node node = parsed.getFirstChild(); while (node!=null) { diff --git a/nb-virtdata/virtdata-api/src/main/resources/virtdata.md b/nb-virtdata/virtdata-api/src/main/resources/virtdata.md index e7caa7d31..8adc7b3a5 100644 --- a/nb-virtdata/virtdata-api/src/main/resources/virtdata.md +++ b/nb-virtdata/virtdata-api/src/main/resources/virtdata.md @@ -1,8 +1,9 @@ --- weight: 225811593 -title: virdata +title: virtdata +source: nb-virtdata/virtdata-api/src/main/resources/virtdata.md --- -# virdata +# virtdata NB5 contains a bundled virtdata app which lets you verify bindings from the command line. It is useful for sanity checking values as well as getting a concurrent performance baseline for diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/Map.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/Map.java index fc1a83ebe..fdb824f09 100644 --- a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/Map.java +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/Map.java @@ -16,8 +16,10 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection; +import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.virtdata.api.annotations.*; +import java.util.Arrays; import java.util.HashMap; import java.util.function.LongFunction; import java.util.function.LongToIntFunction; @@ -61,12 +63,23 @@ public class Map implements LongFunction> { "create a map of size 2, with a specific function for each key and each value"}) @SafeVarargs public Map(LongFunction... objfuncs) { - this.mode = Mode.Tuples; if ((objfuncs.length%2)!=0) { - throw new RuntimeException("An even number of functions must be provided."); + Object testValue = objfuncs[0].apply(0L); + if (testValue instanceof Number n) { + LongFunction[] finalObjfuncs = objfuncs; + this.sizeFunc= l -> ((Number) finalObjfuncs[0].apply(l)).intValue(); + objfuncs = Arrays.copyOfRange(objfuncs, 1, objfuncs.length); + this.mode=Mode.VarSized; + } else { + throw new BasicError("An even number of functions must be provided, unless " + + "the first one produces a numeric value."); + } + } else { + this.mode = Mode.Tuples; + int size = objfuncs.length/2; + sizeFunc=(l) -> size; } int size = objfuncs.length / 2; - sizeFunc=(l) -> size; keyFuncs = new LongFunction[size]; valueFuncs = new LongFunction[size]; for (int i = 0; i < size; i++) { diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/MapTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/MapTest.java index 472d1c807..d3c6c3cd8 100644 --- a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/MapTest.java +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/MapTest.java @@ -16,9 +16,14 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection; +import org.joda.time.DateTime; import org.junit.jupiter.api.Test; +import java.time.Instant; +import java.util.function.LongFunction; + import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class MapTest { @@ -32,26 +37,47 @@ public class MapTest { @Test public void testStringMap() { - StringMap sm = new StringMap((s)->2,(k)->k,(v)->v); + StringMap sm = new StringMap((s) -> 2, (k) -> k, (v) -> v); java.util.Map m2 = sm.apply(11L); - assertThat(m2).containsOnlyKeys("11","12"); - assertThat(m2).containsValues("11","12"); + assertThat(m2).containsOnlyKeys("11", "12"); + assertThat(m2).containsValues("11", "12"); } @Test public void testMapTuple() { - Map mf = new Map(s1 -> (int) s1, k2 -> (int) k2, s2 -> (int) s2, k2 -> (int)k2); + Map mf = new Map(s1 -> (int) s1, k2 -> (int) k2, s2 -> (int) s2, k2 -> (int) k2); java.util.Map mt = mf.apply(37L); - assertThat(mt).containsOnlyKeys(37,38); - assertThat(mt).containsValues(37,38); + assertThat(mt).containsOnlyKeys(37, 38); + assertThat(mt).containsValues(37, 38); } @Test public void testStringMapTuple() { - StringMap mf = new StringMap(s1 -> (int) s1, k2 -> (int) k2, s2 -> (int) s2, k2 -> (int)k2); + StringMap mf = + new StringMap(s1 -> (int) s1, k2 -> (int) k2, s2 -> (int) s2, k2 -> (int) k2); java.util.Map mt = mf.apply(37L); - assertThat(mt).containsOnlyKeys("37","38"); - assertThat(mt).containsValues("37","38"); + assertThat(mt).containsOnlyKeys("37", "38"); + assertThat(mt).containsValues("37", "38"); + } + + @Test + public void testLongFunctionsOnlyWithOddArity() { + LongFunction sizeFunc = l -> (double) (l % 5); + LongFunction keyfunc = String::valueOf; + LongFunction valueFunc = String::valueOf; + Map map = new Map(sizeFunc, keyfunc, valueFunc); + java.util.Map apply = map.apply(3L); + assertThat(apply).isEqualTo(java.util.Map.of("3", "3", "4", "4", "5", "5")); + } + + @Test + public void testLongFunctionsOnlyWithInvalidSizer() { + LongFunction sizeFunc = l -> Instant.ofEpochMilli(1L); + LongFunction keyfunc = String::valueOf; + LongFunction valueFunc = String::valueOf; + assertThatThrownBy(() -> new Map(sizeFunc, keyfunc, valueFunc)).hasMessageContaining("An " + + + "even number of functions must be provided, unless the first one produces a numeric value"); } } diff --git a/nbr-demos/pom.xml b/nbr-demos/pom.xml index 26168d8b3..e8d5576e8 100644 --- a/nbr-demos/pom.xml +++ b/nbr-demos/pom.xml @@ -65,7 +65,7 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.0.0-M6 + 3.0.0 diff --git a/nbr/src/main/resources/export-docs.md b/nbr/src/main/resources/export-docs.md index 2a1aed352..62f395878 100644 --- a/nbr/src/main/resources/export-docs.md +++ b/nbr/src/main/resources/export-docs.md @@ -1,3 +1,7 @@ +--- +source: ./nbr/src/main/resources/export-docs.md +--- + # export-docs This is the built-in app that allows NB5 to export docs for integration into other systems. diff --git a/nbr/src/main/resources/nb_version_info.md b/nbr/src/main/resources/nb_version_info.md index f041ef7e7..a2f4f3719 100644 --- a/nbr/src/main/resources/nb_version_info.md +++ b/nbr/src/main/resources/nb_version_info.md @@ -1,3 +1,6 @@ +--- +source: nbr/src/main/resources/nb_version_info.md +--- # Version Info The version of NB5 which provided some of the docs content is: