Export Docs Improvements and Dependency Updates (#2131)

* Export Docs Improvements and Dependency Updates

* Update build docs job steps
This commit is contained in:
Dave Fisher 2025-01-10 11:17:24 -08:00 committed by GitHub
parent 286c5a78b2
commit 71f66a75fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 167 additions and 69 deletions

View File

@ -81,51 +81,27 @@ jobs:
if: ${{ github.repository == 'nosqlbench/nosqlbench' && github.event_name == 'pull_request' && github.ref_name == 'main' }}
steps:
- name: set git username
run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
- name: set git email
run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
- name: checkout build docs
uses: actions/checkout@v4
with:
repository: nosqlbench/nosqlbench-build-docs
path: build-docs
- name: download exported_docs
uses: actions/download-artifact@v4
with:
name: exported-docs
- run: ls -la
- name: clone nosqlbench-build-docs
env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
run: |
git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs
cd nosqlbench-build-docs
echo "files listing"
find .
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git
git remote -v
- name: import zipfile
run: |
mv exported_docs.zip nosqlbench-build-docs/
- name: set CNAME
run: |
echo "builddocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME
- name: commit changes
run: |
cd nosqlbench-build-docs
git add exported_docs.zip
- name: push changes
env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
- name: overlay docs changes and push
run: |
set -x
cd nosqlbench-build-docs
mv exported_docs.zip build-docs/.
cd build-docs
./doimport.sh
rm exported_docs.zip
git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
git config --global user.name "${{ secrets.NBDROID_NAME }}"
git add .
CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
echo "found $CHANGES to push for doc updates"
if (( $CHANGES > 0 ))
@ -133,4 +109,3 @@ jobs:
git commit -m"docs update for $GITHUB_REF"
git push
fi
echo "push completed"

View File

@ -698,7 +698,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>3.2.1</version>
<version>3.5.0</version>
<executions>
<execution>
<id>enforce-java</id>
@ -716,6 +716,19 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.cyclonedx</groupId>
<artifactId>cyclonedx-maven-plugin</artifactId>
<executions>
<execution>
<phase>install</phase>
<goals>
<goal>makeAggregateBom</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
<pluginManagement>
@ -723,18 +736,17 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.4.2</version>
<version>3.7.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>3.0.0-M7</version>
<version>3.1.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>3.0.1</version>
<version>3.2.7</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -744,53 +756,53 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M8</version>
<version>3.5.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0-M8</version>
<version>3.5.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.4.1</version>
<version>3.11.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.2.1</version>
<version>3.3.1</version>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.13</version>
<version>1.7.0</version>
</plugin>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
<version>4.13.1</version>
<version>4.13.2</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.1.0</version>
<version>3.5.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>3.2.1</version>
<version>3.5.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>3.2.0</version>
<version>3.4.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.3.0</version>
<version>3.3.1</version>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
@ -800,17 +812,17 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.3.0</version>
<version>3.4.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>3.1.0</version>
<version>3.1.3</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>3.0.0</version>
<version>3.1.3</version>
</plugin>
</plugins>
</pluginManagement>
@ -958,7 +970,7 @@
<execution>
<id>attach-javadoc</id>
<goals>
<goal>aggregate-jar</goal>
<goal>aggregate</goal>
</goals>
</execution>

View File

@ -1,6 +1,7 @@
---
weight: 0
title: AMQP
source: nb-adapters/adapter-amqp/src/main/resources/amqp.md
---
- [1. Overview](#1-overview)
- [2. NB AMQP Usage](#2-nb-amqp-usage)

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-cqld4/src/main/resources/cqld4.md
---
# cqld4
This is the newly revamped driver for CQL which uses the DataStax OSS Driver version 4. As

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-cqld4/src/main/resources/cqlgen.md
---
# cqlgen
*cqlgen* is a default CQL workload generator

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-dataapi/src/main/resources/dataapi.md
---
# Data API
## DataStax Data API Adapter

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-diag/src/main/resources/diag.md
---
# diag
This is a diagnostic activity type. Its action simply reports the cycle number and the reporting delay between it's schedule reporting time and the current time. The reporting is interleaved between the threads, with the logical number of reports remaining constant regardless of the thread count.

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-dynamodb/src/main/resources/dynamodb.md
---
# dynamodb
The DynamoDB driver supports a basic set of commands as specified at

View File

@ -0,0 +1,6 @@
---
source: nb-adapters/adapter-example/src/main/resources/example.md
---
# Example
Stub.

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md
---
# Google Spanner driver adapter
The Google Cloud Spanner driver adapter is a NoSQLBench adapter for the `gcp_spanner` driver, a Java driver
for connecting to and performing operations on an instance of a Google Cloud Spanner database.

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-http/src/main/resources/http.md
---
# http
This driver allows you to make http requests using the native HTTP client

View File

@ -1,6 +1,7 @@
---
weight: 0
title: Kafka
source: nb-adapters/adapter-kafka/src/main/resources/kafka.md
---
- [1. Overview](#1-overview)
- [1.1. Example NB Yaml](#11-example-nb-yaml)

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-mongodb/src/main/resources/mongodb.md
---
# MongoDB
This is a driver for MongoDB. It supports the `db.runCommand` API described in [here](https://docs.mongodb.com/manual/reference/command/).

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-neo4j/src/main/resources/neo4j.md
---
# neo4j driver adapter
The neo4j driver adapter is a nb adapter for the Neo4J driver, an open source Java driver for connecting to and

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-pinecone/src/main/resources/pinecone.md
---
# pinecone driver adapter
The pinecone driver adapter is a nb adapter for the pinecone driver, an open source Java driver for connecting to and
performing operations on an instance of a Pinecone Vector database. The driver is hosted on github at

View File

@ -1,8 +1,10 @@
---
weight: 0
title: Pulsar
source: nb-adapters/adapter-pulsar/src/main/resources/pulsar.md
---
# Pulsar
- [1. Overview](#1-overview)
- [1.1. Issues Tracker](#11-issues-tracker)
- [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload)

View File

@ -1,3 +1,7 @@
---
source: nb-adapters/adapter-qdrant/src/main/resources/qdrant.md
---
# qdrant driver adapter
The qdrant driver adapter is a nb adapter for the qdrant driver, an open source Java driver for connecting to and

View File

@ -1,6 +1,7 @@
---
weight: 0
title: S4J
source: nb-adapters/adapter-s4j/src/main/resources/s4j.md
---
- [1. Overview](#1-overview)
- [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload)

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-stdout/src/main/resources/stdout.md
---
# stdout
This is an activity type which allows for the generation of data

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-tcp/src/main/resources/tcpclient.md
---
# tcpclient
**tcpclient acts like a _client push_ version of stdout over TCP**

View File

@ -1,3 +1,6 @@
---
source: nb-adapters/adapter-tcp/src/main/resources/tcpserver.md
---
# tcpserver
**tcpserver acts like a _server push_ version of stdout over TCP**

View File

@ -1,7 +1,10 @@
---
source: nb-adapters/adapter-weaviate/src/main/resources/weaviate.md
---
# Weaviate driver adapter
The Weaviate driver adapter is a NoSQLBench adapter for the `weaviate` driver, an open-source Java driver
for connecting to and performing operations on an instance of a Weaviate vector database. The driver is
for connecting to and performing operations on an instance of a Weaviate vector database. The driver is
leveraged from GitHub at https://github.com/weaviate/java-client.
## Run Commands (Remove prior to merge)
@ -39,8 +42,8 @@ The following parameters must be supplied to the adapter at runtime in order to
instance of the [Weaviate database](https://weaviate.io/developers/weaviate):
* `token` - In order to use the Weaviate database you must have an account. Once the account is created you can [request
an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be
provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and
an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be
provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and
referenced via the `token_file` config option pointing to the path of the file.
* `uri` - When a collection/index is created in the database the URI (aka endpoint) must be specified as well. The adapter will
use the default value of `localhost:8080` if none is provided at runtime. Remember to *not* provide the `https://`

View File

@ -186,7 +186,7 @@
<profile>
<id>adapter-pulsar-include</id>
<activation>
<activeByDefault>false</activeByDefault>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>

View File

@ -269,10 +269,14 @@ public interface DriverAdapter<OPTYPE extends CycleOp<?>, SPACETYPE extends Spac
Optional<Content<?>> bundled_docs = NBIO.local().pathname(
dev_docspath, cp_docspath).first();
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
String markdown = this.getAdapterName() +".md";
System.out.println("Markdown: " + markdown);
Optional<Content<?>> maindoc = NBIO.local().pathname(
"/src/main/resources/" + this.getAdapterName() + ".md",
this.getAdapterName() + ".md"
"/src/main/resources/" + markdown, markdown
).first();
if ( !maindoc.isPresent() ) {
System.out.println("Could not find main document for " + markdown);
}
maindoc.map(Content::asPath).ifPresent(docs::addPath);
return docs.asDocsBinder();
}

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/00_workload_specification.md
---
# Workload Specification
This directory contains the testable specification for workload definitions used by NoSQLBench.

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/01_spectest_formatting.md
---
# SpecTest Formatting
The specifications and examples follow a pattern:

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/02_workload_structure.md
---
# Workload Structure
## Keywords

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/04_op_template_basics.md
---
# Op Templates
Op templates are the recipes provided by users for an operation. These hold examples of payload

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/05_op_template_payloads.md
---
# Op Template Payloads
Payloads in NoSQLBench op templates can be of any type that you can create from bindings, string

View File

@ -1,4 +1,5 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/06_op_template_variations.md
title: Op Template Variations
---

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/07_template_variables.md
---
# Template Variables
Template variables allow for workload descriptions to be parameterized outside the structure of the

View File

@ -1,3 +1,6 @@
---
source: nb-apis/adapters-api/src/main/java/workload_definition/08_parsed_op_api.md
---
# ParsedOp API
In the workload template examples, we show statements as being formed from a string value. This is a

View File

@ -28,6 +28,7 @@ import java.util.Optional;
public class MutableFrontMatter extends LinkedHashMap<String,List<String>> {
String WEIGHT = "weight";
String TITLE = "title";
String SOURCE = "source";
MutableFrontMatter(Map<String,List<String>> data) {
this.putAll(data);
@ -43,6 +44,11 @@ public class MutableFrontMatter extends LinkedHashMap<String,List<String>> {
return Optional.ofNullable(get(WEIGHT)).map(l -> l.get(0)).map(Integer::parseInt).orElse(0);
}
public String getSource() {
assertMaxSingleValued(SOURCE);
return Optional.ofNullable(get(SOURCE)).map(l -> l.get(0)).orElse(null);
}
public void setTitle(String title) {
put(TITLE,List.of(title));
}
@ -51,6 +57,10 @@ public class MutableFrontMatter extends LinkedHashMap<String,List<String>> {
put(WEIGHT,List.of(String.valueOf(weight)));
}
public void setSource(String source) {
put(SOURCE,List.of(source));
}
private void assertMaxSingleValued(String fieldname) {
if (containsKey(fieldname) && get(fieldname).size()>1) {
throw new RuntimeException("Field '" + fieldname + "' can only have zero or one value. It is single-valued.");
@ -60,7 +70,8 @@ public class MutableFrontMatter extends LinkedHashMap<String,List<String>> {
public String asYaml() {
DumpSettings settings = DumpSettings.builder().setDefaultFlowStyle(FlowStyle.BLOCK).build();
Dump dump = new Dump(settings);
return dump.dumpToString(Map.of(TITLE,getTitle(),WEIGHT,getWeight()));
return dump.dumpToString(Map.of(TITLE,getTitle(),WEIGHT,getWeight(),SOURCE,getSource()));
}
}

View File

@ -63,7 +63,16 @@ public class MutableMarkdown {
v.visit(parsed);
Map<String, List<String>> data = v.getData();
this.frontMatter = new MutableFrontMatter(data);
if(data.containsKey("title")) {
this.frontMatter.setTitle(data.get("title").getFirst().toString());
}
if(data.containsKey("source")) {
this.frontMatter.setSource("https://github.com/nosqlbench/nosqlbench/blob/main/"+data.get("source").getFirst().toString());
} else if (this.path != null) {
this.frontMatter.setSource(this.path.getFileName().toString());
} else {
this.frontMatter.setSource("generated by export-docs");
}
if (frontMatter.getTitle()==null || frontMatter.getTitle().isEmpty()) {
Node node = parsed.getFirstChild();
while (node!=null) {

View File

@ -1,8 +1,9 @@
---
weight: 225811593
title: virdata
title: virtdata
source: nb-virtdata/virtdata-api/src/main/resources/virtdata.md
---
# virdata
# virtdata
NB5 contains a bundled virtdata app which lets you verify bindings from the command line.
It is useful for sanity checking values as well as getting a concurrent performance baseline for

View File

@ -65,7 +65,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0-M6</version>
<version>3.0.0</version>
<executions>
<execution>
<goals>

View File

@ -1,3 +1,7 @@
---
source: ./nbr/src/main/resources/export-docs.md
---
# export-docs
This is the built-in app that allows NB5 to export docs for integration into other systems.

View File

@ -1,3 +1,6 @@
---
source: nbr/src/main/resources/nb_version_info.md
---
# Version Info
The version of NB5 which provided some of the docs content is: