IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into listfuncs

Browse Source
This commit is contained in:
Jonathan Shook 2020-07-07 10:51:12 -05:00
commit 0de0cc7d69
784 changed files with 22555 additions and 4070 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/build.yml vendored
View File

@ -2,8 +2,9 @@ name: build
on: on:
push: push:
tags: branches: main
- '!invoke-release' paths:
- '!RELEASENOTES.**'
jobs: jobs:
build: build:

23
.github/workflows/docker.yml vendored
View File

@ -1,23 +0,0 @@
name: Docker Release
on:
push:
tags:
- 'nosqlbench-*'
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Build nosqlbench docker
run: cd nb && docker build -t nosqlbench -f ./Dockerfile-build ./
- name: Publish to Registry
uses: elgohr/Publish-Docker-Github-Action@master
with:
name: nosqlbench/nosqlbench
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
tag_names: true
dockerfile: Dockerfile
workdir: nb

34
.github/workflows/dockerhub.yml vendored
View File

@ -1,34 +0,0 @@
# This is a basic workflow to help you get started with Actions
name: dockerhub
on:
push:
# Sequence of patterns matched against refs/tags
tags:
- 'nosqlbench-*' # Push events to matching nosqlbench-[version]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- name: Login to DockerHub Registry
run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
- name: Get the version
id: vars
run: echo ::set-output name=tag::$(echo ${GITHUB_REF:10})
- name: Build the tagged Docker image
run: docker build ./nb/ --file Dockerfile --tag nosqlbench/nosqlbench:${{steps.vars.outputs.tag}}
- name: Push the tagged Docker image
run: docker push nosqlbench/nosqlbench:${{steps.vars.outputs.tag}}
- name: Build the latest Docker image
run: docker build ./nb/ --file Dockerfile --tag nosqlbench/nosqlbench:latest
- name: Push the latest Docker image
run: docker push nosqlbench/nosqlbench:latest

128
.github/workflows/release.yml vendored
View File

@ -2,35 +2,35 @@ name: release
on: on:
push: push:
branches: paths:
- master - RELEASENOTES.**
tags:
- invoke-release
jobs: jobs:
release: release:
runs-on: ubuntu-18.04 runs-on: ubuntu-18.04
steps: steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v1 - name: checkout repo
uses: actions/checkout@v2
- name: setup java
uses: actions/setup-java@v1
with: with:
java-version: '14' java-version: '14'
java-package: jdk java-package: jdk
architecture: x64 architecture: x64
- name: avoid release loop - name: avoid release loop
run: scripts/avoid-release-loop.sh run: scripts/avoid-release-loop.sh
env: env:
GIT_RELEASE_BOT_NAME: "nb-droid" GIT_RELEASE_BOT_NAME: "nb-droid"
- name: capture tty - name: capture tty for gpg
run: | run: |
echo "::set-env name=TTY::"$(tty) echo "::set-env name=TTY::"$(tty)
echo "::set-env name=GPG_TTY::"$(tty) echo "::set-env name=GPG_TTY::"$(tty)
- name: initialize gpg - name: initialize gpg
# env:
# GPG_TTY: ${TTY}
run: | run: |
set -x set -x
echo "${{ secrets.GITHUB_GPG_KEY }}" | base64 -d > private.key echo "${{ secrets.GITHUB_GPG_KEY }}" | base64 -d > private.key
@ -38,18 +38,27 @@ jobs:
rm ./private.key rm ./private.key
echo "gnupg files:" echo "gnupg files:"
ls -l ~/.gnupg/ ls -l ~/.gnupg/
- name: set git username - name: set git username
run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}" run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
- name: set git email - name: set git email
run: git config --global user.name "${{ secrets.NBDROID_NAME }}" run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
- name: free disk space
run: |
sudo swapoff -a
sudo rm -f /swapfile
sudo apt clean
docker rmi $(docker image ls -aq)
df -h
- name: Cache Maven packages - name: Cache Maven packages
uses: actions/cache@v1 uses: actions/cache@v1
with: with:
path: ~/.m2 path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2 restore-keys: ${{ runner.os }}-m2
- name: read versions - name: read versions
run: | run: |
@ -62,13 +71,23 @@ jobs:
NEXT_SNAPSHOT="${BASE_VERSION}.${NEXT_MINOR_VERSION}-SNAPSHOT" NEXT_SNAPSHOT="${BASE_VERSION}.${NEXT_MINOR_VERSION}-SNAPSHOT"
RELEASE_TAGNAME="nosqlbench-${RELEASE_VERSION}" RELEASE_TAGNAME="nosqlbench-${RELEASE_VERSION}"
echo "::set-env name=NEXT_SNAPSHOT::${NEXT_SNAPSHOT}" echo "::set-env name=NEXT_SNAPSHOT::${NEXT_SNAPSHOT}"
echo "::set-env name=RELEASE_VERSION::${RELEASE_VERSION}" echo "::set-env name=RELEASE_VERSION::${RELEASE_VERSION}"
echo "::set-env name=RELEASE_TAGNAME::${RELEASE_TAGNAME}" echo "::set-env name=RELEASE_TAGNAME::${RELEASE_TAGNAME}"
- name: prepare release summary
id: prepare_summary
run: |
#summary=$(scripts/release-notes.sh)
summary=$(cat RELEASENOTES.md)
summary="${summary//'%'/'%25'}"
summary="${summary//$'\n'/'%0A'}"
summary="${summary//$'\r'/'%0D'}"
echo "::set-output name=release_summary::$summary"
- name: prepare release - name: prepare release
run: scripts/release-prepare.sh run: scripts/release-prepare.sh
env: env:
RELEASE_BRANCH_NAME: "master" RELEASE_BRANCH_NAME: "main"
GIT_RELEASE_BOT_NAME: "nb-droid" GIT_RELEASE_BOT_NAME: "nb-droid"
GIT_RELEASE_BOT_EMAIL: ${{ secrets.GIT_RELEASE_BOT_EMAIL }} GIT_RELEASE_BOT_EMAIL: ${{ secrets.GIT_RELEASE_BOT_EMAIL }}
ACCESS_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }} ACCESS_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
@ -84,8 +103,8 @@ jobs:
- name: perform release - name: perform release
run: scripts/release-perform.sh run: scripts/release-perform.sh
continue-on-error: true continue-on-error: true
env: env:
RELEASE_BRANCH_NAME: "master" RELEASE_BRANCH_NAME: "main"
GIT_RELEASE_BOT_NAME: "nb-droid" GIT_RELEASE_BOT_NAME: "nb-droid"
GIT_RELEASE_BOT_EMAIL: ${{ secrets.GIT_RELEASE_BOT_EMAIL }} GIT_RELEASE_BOT_EMAIL: ${{ secrets.GIT_RELEASE_BOT_EMAIL }}
ACCESS_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }} ACCESS_TOKEN: ${{ secrets.GITHUB_ACCESS_TOKEN }}
@ -97,26 +116,38 @@ jobs:
MAVEN_REPO_SERVER_ID: ${{ secrets.MAVEN_REPO_SERVER_ID }} MAVEN_REPO_SERVER_ID: ${{ secrets.MAVEN_REPO_SERVER_ID }}
MAVEN_REPO_SERVER_USERNAME: ${{ secrets.MVN_REPO_PRIVATE_REPO_USER }} MAVEN_REPO_SERVER_USERNAME: ${{ secrets.MVN_REPO_PRIVATE_REPO_USER }}
MAVEN_REPO_SERVER_PASSWORD: ${{ secrets.MVN_REPO_PRIVATE_REPO_PASSWORD }} MAVEN_REPO_SERVER_PASSWORD: ${{ secrets.MVN_REPO_PRIVATE_REPO_PASSWORD }}
- name: bundle artifacts
- name: upload artifacts
run: | run: |
pwd pwd
ls -l ls -l
mkdir staging && cp nb/target/nb.jar nb/target/nb staging mkdir staging && cp nb/target/nb.jar nb/target/nb staging
- uses: actions/upload-artifact@v1
with: - name: upload artifacts
uses: actions/upload-artifact@v1
with:
name: binaries name: binaries
path: staging path: staging
- name: upload guidebook - name: docker push
uses: docker/build-push-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
repository: nosqlbench/nosqlbench
tags: latest, ${{ env.RELEASE_VERSION }}
tag_with_ref: false
- name: bundle guidebook
run: mkdir guidebook && cp -R nb/target/guidebook guidebook run: mkdir guidebook && cp -R nb/target/guidebook guidebook
- uses: actions/upload-artifact@v1
- name: upload guidebook
uses: actions/upload-artifact@v1
with: with:
name: guidebook name: guidebook
path: guidebook path: guidebook
- name: Create Release - name: create release
id: create_release id: create_release
uses: actions/create-release@v1 uses: actions/create-release@v1
env: env:
@ -126,7 +157,9 @@ jobs:
release_name: Release ${{ env.RELEASE_TAGNAME }} release_name: Release ${{ env.RELEASE_TAGNAME }}
draft: false draft: false
prerelease: false prerelease: false
- name: Upload nb.jar body: ${{ steps.prepare_summary.outputs.release_summary }}
- name: upload nb.jar
id: upload-nb-jar id: upload-nb-jar
uses: actions/upload-release-asset@v1 uses: actions/upload-release-asset@v1
env: env:
@ -136,7 +169,8 @@ jobs:
asset_path: nb/target/nb.jar asset_path: nb/target/nb.jar
asset_name: nb.jar asset_name: nb.jar
asset_content_type: application/octet-stream asset_content_type: application/octet-stream
- name: Upload nb
- name: upload nb binary
id: upload-nb-binary id: upload-nb-binary
uses: actions/upload-release-asset@v1 uses: actions/upload-release-asset@v1
env: env:
@ -147,43 +181,25 @@ jobs:
asset_name: nb asset_name: nb
asset_content_type: application/octet-stream asset_content_type: application/octet-stream
# - name: find latest release
# run: |
# LATEST_GH_RELEASE_ID=$(curl --silent "https://api.github.com/repos/nosqlbench/nosqlbench/releases/latest" | jq -r .id)
# echo "::set-env name=LATEST_GH_RELEASE_ID::${LATEST_GH_RELEASE_ID}"
# - name: upload jar
# uses: actions/upload-release-asset@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# asset_path: nb/target/nb.jar
# asset_name: nb.jar
# asset_content_type: application/octet-stream
# upload_url: https://uploads.github.com/repos/nosqlbench/nosqlbench/releases/${{ env.LATEST_GH_RELEASE_ID }}/assets{?name,label}
# - name: upload binary
# uses: actions/upload-release-asset@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# asset_path: nb/target/nb
# asset_name: nb
# asset_content_type: application/octet-stream
# upload_url: https://uploads.github.com/repos/nosqlbench/nosqlbench/releases/${{ env.LATEST_GH_RELEASE_ID }}/assets{?name,label}
docs: docs:
needs: release needs: release
runs-on: ubuntu-18.04 runs-on: ubuntu-18.04
steps: steps:
- name: set git username - name: set git username
run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}" run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
- name: set git email - name: set git email
run: git config --global user.name "${{ secrets.NBDROID_NAME }}" run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
- name: download guidebook - name: download guidebook
uses: actions/download-artifact@v1 uses: actions/download-artifact@v1
with: with:
name: guidebook name: guidebook
path: guidebook path: guidebook
- run: ls -la - run: ls -la
- name: clone nosqlbench-docs - name: clone nosqlbench-docs
env: env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }} NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
@ -195,6 +211,7 @@ jobs:
find . find .
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-docs.git git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-docs.git
git remote -v git remote -v
- name: push changes - name: push changes
env: env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }} NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
@ -213,6 +230,5 @@ jobs:
git push git push
fi fi

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
.run/**
workshop/** workshop/**
local/** local/**
metrics/** metrics/**

16
.run/NBCLI web foreground dryrun.run.xml Normal file
View File

@ -0,0 +1,16 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="NBCLI web foreground dryrun" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="io.nosqlbench.engine.cli.NBCLI" />
<module name="nb" />
<option name="PROGRAM_PARAMETERS" value="run type=webdriver yaml=local/webtest cycles=1000 cyclerate=100 threads=1 dryrun=true -vv" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="io.nosqlbench.engine.cli.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

2
BUILDING.md
View File

@ -2,7 +2,7 @@
## requirements ## requirements
You need Java 11 or newer to build NoSQLBench. You need Java 12 or newer to build NoSQLBench.
# Building Locally # Building Locally

2
CONTRIBUTING.md
View File

@ -4,7 +4,7 @@ in order to make the contribution process easy and effective for everyone involv
## Code of Conduct ## Code of Conduct
This project follows a [code of conduct](https://github.com/nosqlbench/nosqlbench/CONDUCT.md). This project follows a [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md).
Please read through it at least once if you are going to contribute to our endeavor. Please read through it at least once if you are going to contribute to our endeavor.
## Licensing ## Licensing

2
DOWNLOADS.md
View File

@ -9,7 +9,7 @@ The latest release of NoSQLBench is always available from github releases.
- (be sure to `chmod +x nb` once you download it) - (be sure to `chmod +x nb` once you download it)
- download [the latest release of nb.jar](https://github.com/nosqlbench/nosqlbench/releases/latest/download/nb.jar), a - download [the latest release of nb.jar](https://github.com/nosqlbench/nosqlbench/releases/latest/download/nb.jar), a
single-jar application. single-jar application.
- This requires java 12 or later, make sure your `java -version` command says that you are on Java 12 or later. - This requires java 14 or later, make sure your `java -version` command says that you are on Java 14 or later.
## Docker ## Docker

3
Dockerfile Normal file
View File

@ -0,0 +1,3 @@
FROM openjdk:14-alpine
COPY nb/target/nb.jar nb.jar
ENTRYPOINT ["java","-jar", "nb.jar"]

4
MODULES.md
View File

@ -29,9 +29,9 @@ To make it easier to find the module you are looking for, the following strict
naming conventions have been adopted: naming conventions have been adopted:
- All packages within a module start with `io.nosqlbench`, followed by the module name, - All packages within a module start with `io.nosqlbench`, followed by the module name,
with hyphens converted to dots. For example, the 'engine-api' module contains with hyphens converted to dots. For example, the 'engine-api' module contains
packages in `io.nosqlbench.engine.api`. packages in `io.nosqlbench.engine.api`.
- Modules which implement activity types (high-level protocol drivers) are named `activitytype-...`. - Modules which implement activity types (high-level protocol drivers) are named `driver-...`.
- Modules which provide procedural data generation support are named `virtdata-...`. - Modules which provide procedural data generation support are named `virtdata-...`.
- Modules which provide core runtime logic are named `engine-...`. - Modules which provide core runtime logic are named `engine-...`.
- Project-wide maven defaults are contained in the mvn-defaults module. - Project-wide maven defaults are contained in the mvn-defaults module.

113
README.md
View File

@ -10,88 +10,91 @@
## What is NoSQLBench? ## What is NoSQLBench?
NoSQLBench is a serious performance testing tool for the NoSQL ecosystem. NoSQLBench is a serious performance testing tool for the NoSQL ecosystem. It brings together features and capabilities
It brings together features and capabilities that are not found in any that are not found in any other tool.
other tool.
- You can run common testing workloads directly from the command line. You - You can run common testing workloads directly from the command line. You can start doing this within 5 minutes of
can start doing this within 5 minutes of reading this. reading this.
- You can generate virtual data sets of arbitrary size, with deterministic - You can generate virtual data sets of arbitrary size, with deterministic data and statistically shaped values.
data and statistically shaped values. - You can design custom workloads that emulate your application, contained in a single file, based on statement
- You can design custom workloads that emulate your application, contained templates - no IDE or coding required.
in a single file, based on statement templates - no IDE or coding required. - You can immediately plot your results in a docker and grafana stack on Linux with a single command line option.
- You can immediately plot your results in a docker and grafana stack on Linux - When needed, you can open the access panels and rewire the runtime behavior of NoSQLBench to do advanced testing,
with a single command line option. including a full scripting environment with Javascript.
- When needed, you can open the access panels and rewire the runtime behavior
of NoSQLBench to do advanced testing, including a full scripting environment
with Javascript.
The core machinery of NoSQLBench has been built with attention to detail. The core machinery of NoSQLBench has been built with attention to detail. It has been battle tested within DataStax as a
It has been battle tested within DataStax as a way to help users validate their way to help users validate their data models, baseline system performance, and qualify system designs for scale.
data models, baseline system performance, and qualify system designs for scale.
In short, NoSQLBench wishes to be a programmable power tool for performance In short, NoSQLBench wishes to be a programmable power tool for performance testing. However, it is somewhat generic. It
testing. However, it is somewhat generic. It doesn't know directly about a doesn't know directly about a particular type of system, or protocol. It simply provides a suitable machine harness in
particular type of system, or protocol. It simply provides a suitable machine which to put your drivers and testing logic. If you know how to build a client for a particular kind of system, it will
harness in which to put your drivers and testing logic. If you know how to build let you load it like a plugin and control it dynamically.
a client for a particular kind of system, it will let you load it like a plugin
and control it dynamically.
Initially, NoSQLBench comes with support for CQL, but we would like to see this Initially, NoSQLBench comes with support for CQL, but we would like to see this expanded with contributions from others.
expanded with contributions from others.
## Origins ## Origins
The code in this project comes from multiple sources. The procedural data The code in this project comes from multiple sources. The procedural data generation capability was known before as
generation capability was known before as 'Virtual Data Set'. The core runtime 'Virtual Data Set'. The core runtime and scripting harness was from the 'EngineBlock' project. The CQL support was
and scripting harness was from the 'EngineBlock' project. The CQL support was previously used within DataStax. In March of 2020, DataStax and the project maintainers for these projects decided to
previously used within DataStax. In March of 2020, DataStax and the project put everything into one OSS project in order to make contributions and sharing easier for everyone. Thus, the new
maintainers for these projects decided to put everything into one OSS project project name and structure was launched as nosqlbench.io. NoSQLBench is an independent project that is sponsored by
in order to make contributions and sharing easier for everyone. Thus, the new DataStax.
project name and structure was launched as nosqlbench.io. NoSQLBench is an
independent project that is sponsored by DataStax.
We offer NoSQLBench as a new way of thinking about testing systems. It is not We offer NoSQLBench as a new way of thinking about testing systems. It is not limited to testing only one type of
limited to testing only one type of system. It is our wish to build a community system. It is our wish to build a community of users and practice around this project so that everyone in the NoSQL
of users and practice around this project so that everyone in the NoSQL ecosystem ecosystem can benefit from common concepts and understanding and reliable patterns of use.
can benefit from common concepts and understanding and reliable patterns of use.
## Getting Support
In general, our goals with NoSQLBench are to make the help systems and examples wrap around the users like a suit of
armor, so that they feel capable of doing most things autonomously. Please keep this in mind when looking for personal
support form our community, and help us find those places where the docs are lacking. Maybe you can help us by adding
some missing docs!
### NoSQLBench Slack
There is a new slack channel at nosqlbench.slack.com. In order to access the slack channel, you'll need an invite. You
can get that through
[this simple form](https://docs.google.com/forms/d/e/1FAIpQLSdUOJ8iAPqyxsLfh1nBBsKShI53RAeuzYW4bKExmRMWjj4ufQ/viewform),
which will send you an invite in email: Slack Invite. This is just a simple google form that automates the invite
process.
Please join it if you are a new or existing NoSQLBench user and help us get it going!
## Contributing ## Contributing
We are actively looking for contributors to help make NoSQLBench better. We are actively looking for contributors to help make NoSQLBench better. This is an ambitious project that is just
This is an ambitious project that is just finding its stride. If you want finding its stride. If you want to be part of the next chapter in NoSQLBench development please look at
to be part of the next chapter in NoSQLBench development please look at
[CONTRIBUTING](CONTRIBUTING.md) for ideas, and jump in where you feel comfortable. [CONTRIBUTING](CONTRIBUTING.md) for ideas, and jump in where you feel comfortable.
All contributors are expected to abide by the [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md). All contributors are expected to abide by the [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md).
## License ## License
All of the code in this repository is licensed under the APL version 2. If you contribute All of the code in this repository is licensed under the APL version 2. If you contribute to this project, then you must
to this project, then you must agree to license all of your construbutions under agree to license all of your constributions under this license.
this license.
## System Compatibility ## System Compatibility
This is a Linux targeted tool, as most cloud/nosql testing is done on Linux instances. Some support for other systems is available, but more work is needed to support them fully. Here is what is supported for each: This is a Linux targeted tool, as most cloud/nosql testing is done on Linux instances. Some support for other systems is
available, but more work is needed to support them fully. Here is what is supported for each:
1. on Linux, all features are supported, for both `nb.jar` as well as the appimage binary `nb` 1. on Linux, all features are supported, for both `nb.jar` as well as the appimage binary `nb`
2. on Mac, with `nb.jar` all features are supported, except --docker-metrics 2. on Mac, all features are supported, with `nb.jar`.
3. On Windows, with `nb.jar` all features are supported, except --docker-metrics 3. On Windows, with `nb.jar` all features are supported, except `--docker-metrics`.
## Thanks ## Thanks
[![DataStax Logo](https://www.datastax.com/sites/default/files/content/graphics/logo/DS-logo-2019_1-25percent.png)](http://datastax.com/) [![DataStax Logo](https://www.datastax.com/sites/default/files/content/graphics/logo/DS-logo-2019_1-25percent.png)](http://datastax.com/)
This project is sponsored by [DataStax](http://datstax.com/) -- The always-on, This project is sponsored by [DataStax](http://datastax.com/) -- The always-on, active everywhere, distributed hybrid
active everywhere, distributed hybrid cloud database built on Apache Cassandra™, cloud database built on Apache Cassandra™, and designed from the ground up to run anywhere, on any cloud, in any
and designed from the ground up to run anywhere, on any cloud, in any datacenter, datacenter, and in every possible combination. DataStax delivers the ultimate hybrid and multi-cloud database.
and in every possible combination. DataStax delivers the ultimate hybrid and multi-cloud database.
![YourKit Logo](https://www.yourkit.com/images/yklogo.png) ![YourKit Logo](https://www.yourkit.com/images/yklogo.png)
This project uses tools provided by YourKit, LLC. YourKit supports open source projects This project uses tools provided by YourKit, LLC. YourKit supports open source projects with its full-featured Java
with its full-featured Java Profiler. YourKit, LLC is the creator of Profiler. YourKit, LLC is the creator of <a href="https://www.yourkit.com/java/profiler/">YourKit Java Profiler</a> and
<a href="https://www.yourkit.com/java/profiler/">YourKit Java Profiler</a> <a href="https://www.yourkit.com/.net/profiler/">YourKit .NET Profiler</a>, innovative and intelligent tools for
and <a href="https://www.yourkit.com/.net/profiler/">YourKit .NET Profiler</a>, profiling Java and .NET applications.
innovative and intelligent tools for profiling Java and .NET applications.

14
RELEASENOTES.md Normal file
View File

@ -0,0 +1,14 @@
- a689b4c9 initial NBUI docs
- cd450c17 Fix typo
- b723924e #159 Possible NPE bug in URL resolver
- c5a98420 start of op grammar
- d083483f function adapter handles LongToIntFunction
- b3eecb4d clarify naming of statement fields
- a0dc30a6 clarify default type of parameter
- 5ca319e7 #169 fix template preprocessor regression
- e7daf97d #168 fix parameter types for free params on statements
- 2f083001 Allow easy load balancer configuration for CQL driver #173
- 73464c43 fix scenario parsing bug when named arg contains assignments
- 105c59d3 Bump log4j.version from 2.13.0 to 2.13.3 in /mvn-defaults
- 8725cc40 Fix MongoAction incorrect error message
- 7f2c2e31 mongodb: Update yaml

46
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/core/CqlActivityType.java
View File

@ -1,46 +0,0 @@
package io.nosqlbench.activitytype.cql.core;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.annotations.Service;
import java.util.Optional;
@Service(ActivityType.class)
public class CqlActivityType implements ActivityType<CqlActivity> {
public String getName() {
return "cql";
}
@Override
public CqlActivity getActivity(ActivityDef activityDef) {
Optional<String> yaml = activityDef.getParams().getOptionalString("yaml", "workload");
// sanity check that we have a yaml parameter, which contains our statements and bindings
if (yaml.isEmpty()) {
throw new RuntimeException("Currently, the cql activity type requires yaml/workload activity parameter.");
}
// allow shortcut: yaml parameter provide the default alias name
if (activityDef.getAlias().equals(ActivityDef.DEFAULT_ALIAS)) {
activityDef.getParams().set("alias",yaml.get());
}
return new CqlActivity(activityDef);
}
/**
* Returns the per-activity level dispenser. The ActionDispenser can then dispense
* per-thread actions within the activity instance.
* @param activity The activity instance which will parameterize this action
*/
@Override
public ActionDispenser getActionDispenser(CqlActivity activity) {
return new CqlActionDispenser(activity);
}
}

19
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/statements/binders/SimpleStatementValuesBinder.java
View File

@ -1,19 +0,0 @@
package io.nosqlbench.activitytype.cql.statements.binders;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import io.nosqlbench.virtdata.core.bindings.ValuesArrayBinder;
/**
* This binder is not meant to be used with anything but DDL or statements
* which should not be trying to parameterize values in general. If this changes,
* support will be added for parameterized values here.
*/
public class SimpleStatementValuesBinder
implements ValuesArrayBinder<SimpleStatement, Statement> {
@Override
public Statement bindValues(SimpleStatement context, Object[] values) {
return new SimpleStatement(context.getQueryString(), values);
}
}

125
activitytype-http/src/main/java/io/nosqlbench/activitytype/http/HttpAction.java
View File

@ -1,125 +0,0 @@
package io.nosqlbench.activitytype.http;
import com.codahale.metrics.Timer;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.*;
import java.util.concurrent.TimeUnit;
public class HttpAction implements SyncAction {
private final static Logger logger = LoggerFactory.getLogger(HttpAction.class);
private final HttpActivity httpActivity;
private final int slot;
private int maxTries = 1;
private boolean showstmts;
private OpSequence<StringBindings> sequencer;
public HttpAction(ActivityDef activityDef, int slot, HttpActivity httpActivity) {
this.slot = slot;
this.httpActivity = httpActivity;
}
@Override
public void init() {
this.sequencer = httpActivity.getOpSequence();
}
@Override
public int runCycle(long cycleValue) {
StringBindings stringBindings;
String statement = null;
InputStream result = null;
try (Timer.Context bindTime = httpActivity.bindTimer.time()) {
stringBindings = sequencer.get(cycleValue);
statement = stringBindings.bind(cycleValue);
String[] splitStatement = statement.split("\\?");
String path, query;
String host = httpActivity.getHosts()[(int) cycleValue % httpActivity.getHosts().length];
path = splitStatement[0];
query = "";
if (splitStatement.length >= 2) {
query = splitStatement[1];
}
URI uri = new URI(
"http",
null,
host,
httpActivity.getPort(),
path,
query,
null);
statement = uri.toString();
showstmts = httpActivity.getShowstmts();
if (showstmts) {
logger.info("STMT(cycle=" + cycleValue + "):\n" + statement);
}
} catch (URISyntaxException e) {
e.printStackTrace();
}
long nanoStartTime=System.nanoTime();
int tries = 0;
while (tries < maxTries) {
tries++;
try (Timer.Context executeTime = httpActivity.executeTimer.time()) {
URL url = new URL(statement);
//
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
result = conn.getInputStream();
} catch (Exception e) {
throw new RuntimeException("Error writing output:" + e, e);
}
Timer.Context resultTime = httpActivity.resultTimer.time();
try {
StringBuilder res = new StringBuilder();
BufferedReader rd = new BufferedReader(new InputStreamReader(result));
String line;
while ((line = rd.readLine()) != null) {
res.append(line);
}
rd.close();
} catch (Exception e) {
long resultNanos = resultTime.stop();
resultTime=null;
} finally {
if (resultTime!=null) {
resultTime.stop();
}
}
}
long resultNanos=System.nanoTime() - nanoStartTime;
httpActivity.resultSuccessTimer.update(resultNanos, TimeUnit.NANOSECONDS);
return 0;
}
protected HttpActivity getHttpActivity() {
return httpActivity;
}
}

52
activitytype-http/src/main/resources/http.md
View File

@ -1,52 +0,0 @@
# http activity type
This activity type allows for basic HTTP requests.
As of this release, only GET requests are supported.
## Example activity definitions
Run an http activity named 'http-test', with definitions from activities/http-google.yaml:
~~~
... driver=http workload=http-google
~~~
This last example shows that the cycle range is [inclusive..exclusive),
to allow for stacking test intervals. This is standard across all
activity types.
## stdout ActivityType Parameters
- **host** - The hosts to send requests to. The hosts are selected in
round-robin fashion.
(default: localhost)
- **workload** - The workload definition file which holds the schema and statement defs.
(no default, required)
- **cycles** - standard, however the activity type will default
this to however many statements are included in the current
activity, after tag filtering, etc.
(default: 0)
- **alias** - this is a standard nosqlbench parameter
(default: derived from the workload name)
## Configuration
This activity type uses the uniform yaml configuration format.
For more details on this format, please refer to the
[Standard YAML Format](http://docs.nosqlbench.io/user-guide/standard_yaml/)
## Configuration Parameters
- **ratio** - If a statement has this param defined, then it determines
whether or not to automatically add a missing newline for that statement
only. If this is not defined for a statement, then the activity-level
parameter takes precedence.
- **seq** - The statement sequencer scheme.
(default: bucket)
## Statement Format
The statement format for this activity type is a simple string. Tokens between
curly braces are used to refer to binding names, as in the following example:
statements:
- "/{path}?{queryparam1}"

6
activitytype-http/src/test/resources/activities/http-google.yaml
View File

@ -1,6 +0,0 @@
tags:
type: google
params:
requestType: GET
statements:
- /

6
devdocs/INDEX.md Normal file
View File

@ -0,0 +1,6 @@
# Developer Docs and RFCs
This directory is where new ideas or developer guides will be organized for now. They may eventually find their way into
a developer-focused documentation section.

107
devdocs/devguide/error_handling.md Normal file
View File

@ -0,0 +1,107 @@
# Error Handling in NoSQLBench
This guide acts as a design template for how the error handling should be made consistent throughout
all of NoSQLBench and supported drivers.
## Scopes of Execution
![Scopes](scopes.png)
### Process
A NoSQLBench process is active when you start a scenario from the command line, or when you start it
in docserver or some other daemon mode.
### Scenario
When a NoSqlBench process runs, it can execute scenario scripts in ECMAScript.
These are called scenarios. Each scenario runs independently of any others
in the current process.
Scenario scripts are completely unlimited in what they can do.
### Activity
Scenario scripts may use the scripting API to start protocol specific activities which run over a
range of cycles. These activities run independently of each other within the scenario.
### Cycle
Activities run like flywheels over a range of cycle values. Each cycle is responsible for initiating
a single operation with the help of a driver. In this context, _cycle_ means two things:
1. It is the specific value on the number line which is used as the seed value for all synthesized
operations.
2. It is the logic which uses this cycle value to determine which operation to execute, what
synthetic data to bind into it, and how to combine them together into a native operation for the
target system.
### Operation
Within a cycle, an operation will be submitted to a native driver or target system, and the result
will be scrutinized. It may be retried, and further operations based on the first one may be
injected additionally to run within the same cycle.
Process
Scenario
Activity
Cycle
Operation
## Handling Errors
### Basic Errors
*BasicErrors* Are errors for which NoSQLBench knows the exact reason for it happening and can thus
inform the user with a direct error message and nothing else. Anywhere a specific type of error can
be caught which gives the user a direct understanding of what cause the error or how to correct it,
you should use a BasicError. All of the exception handling logic in NoSQLBench should recognize the
BasicError exception type and allow it to propoage unmodified to the top-most exception handler.
This allows consistent handling for these errors so that users don't get spammed with stack traces
and other distractions when they are not needed.
## Checked Exceptions
Checked exceptions are not followed dogmatically as a programming doctrine in NoSQLBench. The
reasons are more practical than philosophical, however. The gist is that littering contextual error
handlers all over the place for checked exceptions would over-complicate the layering of exception
handling rather than simplify it. Checked exceptions can force non-trivial code bases to have an
arbitrarily higher surface area when simpler modal exception handlers would suffice. Thus, if you
are having to deal with a checked exception and aren't sure how to handle it, it is generally OK to
wrap it in a RuntimeException and rethrow it. Some may disagree on this approach, and for those
developers, we are happy to take pull requests that make genuine improvements in this area.
## Error Handlers
Because NoSQLBench is a tool for testing things, it is important that the user have the ability to
customize the exception handling behavior according to testing criteria. This means both the ability
to say when to count certain outcomes as errors as well as the ability to retry for possibly
intermittent failures, and to communicate the status of errors clearly for consumption by other
users or systems.
The options that users are offered for handling errors with the CQL driver, for example are routable
by error type to any of:
1. stop
2. warn
3. retry
4. histogram
5. count
6. ignore
Where each error handler drops through all the rest once the error handling logic is invoked. The
user specifies at which level they want to handle specific types of errors and whether to consider
certain types of errors ad retryable or not.
Each driver needs to support this level of configuration. A simpler and more consistent API should
be built to make this easy for driver implementors.
Error handlers of this type are only expected at the operational level within a cycle. That is, the
error handling in the rest of the NoSQLBench machinery need not be so configurable. Thus, the error
handling semantics need to be dealt with on a driver-specific level.
## Scripting Errors

BIN
devdocs/devguide/scopes.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 54 KiB

519
devdocs/devguide/scopes.svg Normal file
View File

@ -0,0 +1,519 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
sodipodi:docname="scopes.svg"
inkscape:version="1.1-dev (1:1.0+devel+202006301644+a9d5e92)"
id="svg928"
version="1.1"
viewBox="0 0 431.80001 279.4"
height="11in"
width="17in">
<defs
id="defs924" />
<sodipodi:namedview
inkscape:window-maximized="1"
inkscape:window-y="2160"
inkscape:window-x="0"
inkscape:window-height="2007"
inkscape:window-width="3840"
inkscape:object-paths="true"
inkscape:snap-intersection-paths="true"
inkscape:snap-smooth-nodes="true"
inkscape:snap-object-midpoints="true"
units="in"
showgrid="true"
inkscape:current-layer="g1094"
inkscape:document-units="mm"
inkscape:cy="446.75016"
inkscape:cx="558.31404"
inkscape:zoom="2.8284271"
inkscape:pageshadow="2"
inkscape:pageopacity="0.0"
borderopacity="1.0"
bordercolor="#666666"
pagecolor="#ffffff"
id="base">
<inkscape:grid
opacity="0.07058824"
color="#3f3fff"
units="in"
spacingy="1.5875"
spacingx="1.5875"
empspacing="16"
id="grid987"
type="xygrid" />
<inkscape:grid
dotted="false"
empspacing="4"
spacingy="6.3500001"
spacingx="6.3500001"
units="in"
id="grid989"
type="xygrid" />
</sodipodi:namedview>
<g
id="layer1"
inkscape:groupmode="layer"
inkscape:label="Layer 1">
<g
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
id="g1206">
<rect
style="fill:#ffffff;stroke:#000000;stroke-width:1;stroke-opacity:0.97228;stop-color:#000000;font-variation-settings:normal;opacity:1;vector-effect:none;fill-opacity:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stop-opacity:1"
id="rect993"
width="127"
height="101.6"
x="127"
y="76.199997" />
<rect
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:0.5;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.97228;stop-color:#000000"
id="rect995"
width="50.799995"
height="12.700005"
x="133.35001"
y="69.849998" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:8.46667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
x="135.44458"
y="78.445641"
id="text999"><tspan
sodipodi:role="line"
id="tspan997"
style="font-size:8.46667px;stroke-width:0.264583"
x="135.44458"
y="78.445641">NB Process</tspan></text>
</g>
<g
transform="translate(-6.3500031,-9.525)"
id="g1094">
<rect
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="rect993-6-7"
width="111.12501"
height="69.849998"
x="139.7"
y="101.6" />
<rect
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.97228;stop-color:#000000"
id="rect995-5-3"
width="38.099979"
height="12.700002"
x="146.05"
y="95.25" />
<text
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:8.46667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
x="148.25134"
y="103.874"
id="text999-5-2"><tspan
sodipodi:role="line"
id="tspan997-6-9"
style="font-size:8.46667px;stroke-width:0.264583"
x="148.25134"
y="103.874">Scenario</tspan></text>
</g>
<g
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
transform="translate(-3.1750031,-3.1750046)"
id="g1094-1">
<rect
y="101.6"
x="139.7"
height="71.437508"
width="111.12501"
id="rect993-6-7-2"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<rect
y="95.25"
x="146.05"
height="12.700002"
width="38.099979"
id="rect995-5-3-7"
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.97228;stop-color:#000000" />
<text
id="text999-5-2-0"
y="103.874"
x="148.25134"
style="font-style:normal;font-weight:normal;font-size:8.46667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
xml:space="preserve"><tspan
y="103.874"
x="148.25134"
style="font-size:8.46667px;stroke-width:0.264583"
id="tspan997-6-9-9"
sodipodi:role="line">Scenario</tspan></text>
</g>
<g
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
transform="translate(-2.8796539,4.4671513)"
id="g1532">
<g
id="g1094-1-3-1"
transform="translate(39.687508,2.3812527)">
<rect
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="rect993-6-7-2-6-8"
width="60.325005"
height="33.337505"
x="139.7"
y="101.6" />
<rect
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.97228;stop-color:#000000"
id="rect995-5-3-7-0-7"
width="38.100002"
height="9.5250034"
x="146.05"
y="96.837502" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
x="148.25134"
y="103.874"
id="text999-5-2-0-6-9"><tspan
sodipodi:role="line"
id="tspan997-6-9-9-2-2"
style="font-size:7.05556px;stroke-width:0.264583"
x="148.25134"
y="103.874">Activity</tspan></text>
</g>
<g
id="g1414-0"
transform="matrix(0.60465102,0,0,0.60465102,181.41803,57.888391)">
<path
id="path1416-3"
d="M 9.6261342,117.475 H 82.549999"
style="color:#000000;fill:none;stroke-width:2" />
<path
id="path1418-6"
d="m 9.6269531,116.47461 v 2 H 82.550781 v -2 z"
style="color:#000000;fill:#000000;stroke-width:2" />
</g>
<g
id="g1266-1"
transform="matrix(0.60465102,0,0,0.60465102,181.7503,57.556122)">
<circle
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1208-06"
cx="28.575001"
cy="117.475"
r="3.175" />
<ellipse
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1210-3"
cx="41.275002"
cy="117.475"
rx="3.175"
ry="3.1749992" />
<ellipse
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1212-20"
cx="53.974998"
cy="117.47501"
rx="3.1749992"
ry="3.1750007" />
<circle
r="3.175"
cy="109.5375"
cx="30.1625"
id="path1208-0-6"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<ellipse
ry="3.1749992"
rx="3.175"
cy="109.5375"
cx="55.5625"
id="path1210-2-1"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<ellipse
ry="3.1750007"
rx="3.1749992"
cy="117.47501"
cx="66.675003"
id="path1212-3-5"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<circle
r="3.1749992"
cy="93.662498"
cx="71.4375"
id="path1214-7-5"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<circle
r="3.175"
cy="101.6"
cx="44.450001"
id="path1208-5-4"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#ec748c;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<ellipse
ry="3.1749992"
rx="3.175"
cy="109.5375"
cx="42.862499"
id="path1210-9-7"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<ellipse
ry="3.1750007"
rx="3.1749992"
cy="109.5375"
cx="68.262497"
id="path1212-2-6"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<circle
r="3.1749992"
cy="101.6"
cx="69.849998"
id="path1214-2-5"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
</g>
<path
id="path1268-6"
d="m 185.25756,127 1.91977,1.91977 -1.91977,1.91977 3.83953,-1.91977 z"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.15998px;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;stop-color:#000000;stop-opacity:1" />
<path
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.15998px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
d="m 229.4122,127 1.91977,1.91977 -1.91977,1.91977 3.83953,-1.91977 z"
id="path1268-8-9" />
</g>
<g
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
id="g1532-3"
transform="translate(3.1749859,16.668745)">
<g
transform="translate(39.687508,2.3812527)"
id="g1094-1-3-1-7">
<rect
y="101.6"
x="139.7"
height="33.337505"
width="60.325005"
id="rect993-6-7-2-6-8-4"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<rect
y="96.837502"
x="146.05"
height="9.5250034"
width="38.100002"
id="rect995-5-3-7-0-7-5"
style="fill:#ffffff;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:0.97228;stop-color:#000000" />
<text
id="text999-5-2-0-6-9-2"
y="103.874"
x="148.25134"
style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
xml:space="preserve"><tspan
y="103.874"
x="148.25134"
style="font-size:7.05556px;stroke-width:0.264583"
id="tspan997-6-9-9-2-2-5"
sodipodi:role="line">Activity</tspan></text>
</g>
<g
transform="matrix(0.60465102,0,0,0.60465102,181.41803,57.888391)"
id="g1414-0-4">
<path
style="color:#000000;fill:none;stroke-width:2"
d="M 9.6261342,117.475 H 82.549999"
id="path1416-3-7" />
<path
style="color:#000000;fill:#000000;stroke-width:2"
d="m 9.6269531,116.47461 v 2 H 82.550781 v -2 z"
id="path1418-6-4" />
</g>
<g
transform="matrix(0.60465102,0,0,0.60465102,181.7503,57.556122)"
id="g1266-1-4">
<circle
r="3.175"
cy="117.475"
cx="28.575001"
id="path1208-06-3"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<ellipse
ry="3.1749992"
rx="3.175"
cy="117.475"
cx="41.275002"
id="path1210-3-0"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<ellipse
ry="3.1750007"
rx="3.1749992"
cy="117.47501"
cx="53.974998"
id="path1212-20-7"
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1" />
<circle
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1208-0-6-8"
cx="30.1625"
cy="109.5375"
r="3.175" />
<ellipse
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1210-2-1-6"
cx="55.5625"
cy="109.5375"
rx="3.175"
ry="3.1749992" />
<ellipse
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1212-3-5-8"
cx="66.675003"
cy="117.47501"
rx="3.1749992"
ry="3.1750007" />
<circle
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1214-7-5-8"
cx="71.4375"
cy="93.662498"
r="3.1749992" />
<circle
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#ec748c;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1208-5-4-4"
cx="44.450001"
cy="101.6"
r="3.175" />
<ellipse
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1210-9-7-3"
cx="42.862499"
cy="109.5375"
rx="3.175"
ry="3.1749992" />
<ellipse
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#b0b0b0;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1212-2-6-1"
cx="68.262497"
cy="109.5375"
rx="3.1749992"
ry="3.1750007" />
<circle
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#bcea76;fill-opacity:1;stroke:#000000;stroke-width:0.499999;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:0.97228;stop-color:#000000;stop-opacity:1"
id="path1214-2-5-4"
cx="69.849998"
cy="101.6"
r="3.1749992" />
<text
id="text1720"
y="119.1413"
x="28.603626"
style="font-style:normal;font-weight:normal;font-size:4.66752px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.43758"
xml:space="preserve"><tspan
y="119.1413"
x="28.603626"
style="font-size:4.66752px;text-align:center;text-anchor:middle;stroke-width:0.43758"
id="tspan1718"
sodipodi:role="line">3</tspan></text>
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:4.66754px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.43758"
x="41.273289"
y="119.15064"
id="text1720-6"><tspan
sodipodi:role="line"
id="tspan1718-8"
style="font-size:4.66754px;text-align:center;text-anchor:middle;stroke-width:0.43758"
x="41.273289"
y="119.15064">4</tspan></text>
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:4.66754px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.43758"
x="53.961617"
y="119.11799"
id="text1720-9"><tspan
sodipodi:role="line"
id="tspan1718-2"
style="font-size:4.66754px;text-align:center;text-anchor:middle;stroke-width:0.43758"
x="53.961617"
y="119.11799">5</tspan></text>
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:4.66754px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.43758"
x="66.656952"
y="119.14133"
id="text1720-66"><tspan
sodipodi:role="line"
id="tspan1718-4"
style="font-size:4.66754px;text-align:center;text-anchor:middle;stroke-width:0.43758"
x="66.656952"
y="119.14133">6</tspan></text>
<text
id="text1720-6-9"
y="123.45961"
x="16.02532"
style="font-style:normal;font-weight:normal;font-size:4.66754px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.43758"
xml:space="preserve"><tspan
y="123.45961"
x="16.02532"
style="font-size:4.66754px;text-align:center;text-anchor:middle;stroke-width:0.43758"
id="tspan1718-8-5"
sodipodi:role="line">cycles</tspan></text>
</g>
<path
style="font-variation-settings:normal;opacity:1;vector-effect:none;fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.15998px;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;stop-color:#000000;stop-opacity:1"
d="m 185.25756,127 1.91977,1.91977 -1.91977,1.91977 3.83953,-1.91977 z"
id="path1268-6-9" />
<path
id="path1268-8-9-2"
d="m 229.4122,127 1.91977,1.91977 -1.91977,1.91977 3.83953,-1.91977 z"
style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.15998px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
</g>
<g
inkscape:export-ydpi="158.75"
inkscape:export-xdpi="158.75"
style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.264583"
id="text1652"
aria-label="script">
<g
transform="translate(97.928291,-36.93438)"
id="g1684">
<path
sodipodi:nodetypes="cccccsccccsssscc"
id="path1632"
style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
d="m 47.625001,184.15 h 19.05 M 47.624999,152.4 h 19.05 m 10e-7,0 c 1.5875,0 3.175003,4.7625 3.175,6.35 0,1.5875 -6.35,17.4625 -6.35,19.05 m 3.175,6.35 c 1.5875,0 3.175,-6.35 3.175,-6.35 -6.35,0 -12.700001,0 -19.050001,0 0,0 -1.5875,6.35 -3.175,6.35 -1.5875,0 -3.175,-4.7625 -3.175,-6.35 0,-1.5875 6.35,-17.4625 6.35,-19.05 3e-6,-1.5875 -1.5875,-6.35 -3.175,-6.35 -1.5875,0 -3.175,4.7625 -3.175,6.35 h 6.35" />
<g
id="g1673"
transform="matrix(1,0,-0.43100982,1,96.608751,0.70745)">
<path
id="path1654"
style="font-size:5.64444px;stroke-width:0.264583"
d="m 27.528533,162.62439 q 0,0.44026 -0.327378,0.66604 -0.327377,0.22578 -0.880532,0.22578 -0.316089,0 -0.547511,-0.0508 -0.225777,-0.0508 -0.400755,-0.14111 v -0.45156 q 0.180622,0.0903 0.434622,0.16933 0.259644,0.0734 0.524933,0.0734 0.378177,0 0.54751,-0.11853 0.169334,-0.12418 0.169334,-0.32738 0,-0.11289 -0.06209,-0.2032 -0.06209,-0.0903 -0.225778,-0.18062 -0.158044,-0.0903 -0.4572,-0.2032 -0.29351,-0.11289 -0.502355,-0.22578 -0.208844,-0.11289 -0.321733,-0.27093 -0.112889,-0.15805 -0.112889,-0.4064 0,-0.38382 0.310445,-0.59267 0.316088,-0.20884 0.824088,-0.20884 0.276577,0 0.513644,0.0564 0.242711,0.0508 0.451555,0.14676 l -0.169333,0.39511 q -0.191911,-0.079 -0.400755,-0.13547 -0.208845,-0.0564 -0.428978,-0.0564 -0.3048,0 -0.468488,0.1016 -0.158045,0.096 -0.158045,0.26529 0,0.12417 0.07338,0.21449 0.07338,0.0847 0.242711,0.16933 0.174978,0.079 0.462844,0.19191 0.287867,0.10724 0.491066,0.22013 0.2032,0.11289 0.310445,0.27658 0.107244,0.15804 0.107244,0.40076 z" />
<path
id="path1656"
style="font-size:5.64444px;stroke-width:0.264583"
d="m 29.473131,163.51621 q -0.400755,0 -0.716844,-0.16369 -0.310444,-0.16369 -0.491066,-0.508 -0.174978,-0.34431 -0.174978,-0.88053 0,-0.5588 0.186266,-0.90876 0.186267,-0.34995 0.502356,-0.51364 0.321733,-0.16369 0.728132,-0.16369 0.231422,0 0.445911,0.0508 0.214489,0.0452 0.349956,0.11289 l -0.1524,0.41204 q -0.135467,-0.0508 -0.316089,-0.096 -0.180622,-0.0452 -0.338666,-0.0452 -0.891822,0 -0.891822,1.14582 0,0.54751 0.214489,0.84102 0.220133,0.28787 0.64911,0.28787 0.248356,0 0.434622,-0.0508 0.191911,-0.0508 0.349956,-0.12418 v 0.44027 q -0.1524,0.079 -0.338667,0.11853 -0.180622,0.0452 -0.440266,0.0452 z" />
<path
id="path1658"
style="font-size:5.64444px;stroke-width:0.264583"
d="m 32.382663,160.3779 q 0.08467,0 0.180622,0.0113 0.1016,0.006 0.174978,0.0226 l -0.06209,0.4572 q -0.07338,-0.0169 -0.163689,-0.0282 -0.08467,-0.0113 -0.163689,-0.0113 -0.231422,0 -0.434622,0.12983 -0.203199,0.12417 -0.327377,0.3556 -0.118533,0.22577 -0.118533,0.53057 v 1.61431 h -0.496711 v -3.02542 h 0.4064 l 0.05644,0.55316 h 0.02258 q 0.146755,-0.24836 0.378177,-0.42898 0.231422,-0.18062 0.547511,-0.18062 z" />
<path
id="path1660"
style="font-size:5.64444px;stroke-width:0.264583"
d="m 33.55168,159.29981 q 0.112889,0 0.197555,0.079 0.09031,0.0734 0.09031,0.23707 0,0.15804 -0.09031,0.23707 -0.08467,0.079 -0.197555,0.079 -0.124178,0 -0.208844,-0.079 -0.08467,-0.079 -0.08467,-0.23707 0,-0.16369 0.08467,-0.23707 0.08467,-0.079 0.208844,-0.079 z m 0.242711,1.13453 v 3.02542 H 33.29768 v -3.02542 z" />
<path
id="path1662"
style="font-size:5.64444px;stroke-width:0.264583"
d="m 36.192219,160.3779 q 0.5588,0 0.897466,0.38946 0.344311,0.38947 0.344311,1.17405 0,0.77329 -0.344311,1.17404 -0.338666,0.40076 -0.90311,0.40076 -0.349956,0 -0.581378,-0.12983 -0.225777,-0.13546 -0.355599,-0.31044 h -0.03387 q 0.01129,0.096 0.02258,0.24271 0.01129,0.14676 0.01129,0.254 v 1.24178 h -0.496711 v -4.38009 h 0.4064 l 0.06773,0.41205 h 0.02258 q 0.135466,-0.19756 0.355599,-0.33303 0.220133,-0.13546 0.587022,-0.13546 z m -0.09031,0.41769 q -0.462844,0 -0.654755,0.25964 -0.186267,0.25965 -0.197555,0.79022 v 0.096 q 0,0.5588 0.180622,0.8636 0.186266,0.29915 0.682977,0.29915 0.276578,0 0.451555,-0.1524 0.180622,-0.1524 0.265289,-0.41204 0.09031,-0.26529 0.09031,-0.60396 0,-0.51928 -0.2032,-0.82973 -0.197555,-0.31044 -0.615244,-0.31044 z" />
<path
id="path1664"
style="font-size:5.64444px;stroke-width:0.264583"
d="m 39.235896,163.10981 q 0.112889,0 0.231422,-0.0169 0.118533,-0.0226 0.191911,-0.0452 v 0.37818 q -0.07902,0.0395 -0.225778,0.0621 -0.146755,0.0282 -0.282222,0.0282 -0.237066,0 -0.440266,-0.079 -0.197555,-0.0847 -0.321733,-0.28786 -0.124178,-0.2032 -0.124178,-0.57009 v -1.76107 h -0.428977 v -0.23706 l 0.434622,-0.19756 0.197555,-0.64346 h 0.293511 v 0.69426 h 0.874888 v 0.38382 h -0.874888 v 1.74978 q 0,0.27658 0.129822,0.41205 0.135467,0.12982 0.344311,0.12982 z" />
</g>
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 29 KiB

99
devdocs/docstructure/bundled_docs.md Normal file
View File

@ -0,0 +1,99 @@
# Bundled Docs
In order to keep the structure of NoSQLBench modular enough to allow for easy extension by contributors, yet cohesive in
how it presents documentation and features to users, it is necessary to provide internal services which aggregate
content by subject matter into a consumable whole that can be used by the documentation system.
# MarkdownDocs Service
The primary markdown service that is meant to be consumed by the documetnation system is known simply as
MarkdownDocs
Static methods on this class will provide all of the markdown content in pre-baked and organized form. The markdown
service is responsible for reading all the raw markdown sources and organizing their content into a single cohesive
structure. MardownDocs finds all content that is provided by individual MarkdownProvider services, as described below.
All of the rules for how raw markdown content is to be combined are owned by the MarkdownDocs service.
The MarkdownDocs service relies on SPI published services which provide raw markdown sources as described below.
# RawMarkdownSource Services
The `RawMarkdownSource` service is responsible for bundling the raw markdown for a path within a NoSQLBench module. Each
module that wishes to publish markdown docs to users must provide one or more RawMarkdownSource services via SPI. This
is most easily done with a `@Service(RawMarkdownSource.class)` annotation.
## RawMarkdownSource endpoints provide Content
Each instance of a RawMarkdownSource service provides all of the individual markdown files it finds indirectly as
io.nosqlbench.nb.api.content.Content, which allows the internal file content to be read appropriately regardless of
whether it comes from a classpath resource stream, a file on disk, or even a dynamic source like function metadata.
## RawMarkdownSources Aggregator
A service aggregator called RawMarkdownSources provides easy access to all raw markdown sources provided by all
published instances of the service.
# Front Matter Interpretation
There is a set of rules observed by MarkdownDocs for repacking markdown for structured display. These rules are largely
driven by front matter.
## Doc Scope
There are three doc scopes that can be added to source markdown via the `scopes` front matter.
* `cli` - The source content should be included for command-line searching and viewing.
* `web` - The source content should be included for static web documentation.
* `app` - The source content should be included in accompanying documentation for web applications.
* `all` - This is the default scope which includes all of the above.
If no scopes are provided, then a special scope `any` is assigned to source content.
__ THIS IS A WORK IN PROGRESS __
## Topic Names
The `topic` property determines
1. Front matter may be sanity checked for unrecognized properties.
2. All front matter that is considered is required to have at least one topic value.
3. Topic values which contain `, ` or `; ` patterns are auto-split into multiple topics.
4. Topics can be hierarchical. Topics in the form of `cat1/cat2/topicfoo` are considered nested topics, with the
containing layer being considered a category. The right most word is considered the basic topic name. This means that
in the above topic name, `cat1` is a topic category containing the `cat2` topic category, which contains the topic
`topicfoo`.
5. *Topic Expansion* - A topic entry which starts with a caret `^`, contains either of '.*', '.+', or ends with a `$` is
considered a wildcard topic. It will be treated as a topic pattern which will be compared to known topics. When it
matches another topic, the matched topic is added to the virtualized topic list of the owning item.
6. `aggregations` are used to physically aggregate content from matching topics onto a markdown source:
1. Each aggregation is a pattern that is tested against all topics after topic expansion.
2. When a source item is matched to an aggregation,
3. wildcards, except that they
cause all matching topics to be aggregated onto the body of the owning markdown source.
4. All topics (after topicin order determined by weight. Aggregations are indicated with an `aggregation` property.
regations are split on commas and semicolons as above, and are always considered patterns for matching. Thus,
aggregation with none of the regex indicators above will only match topics with the same literal pattern.
7. Front matter will be provided with topical aggregations included, with the following conditions:
* aggregations properties are elided from the repacked view. Instead, an `included` header is added which lists all
of the included topics.
## Composite Markdown
When aggregations occur, the resulting markdown that is produces is simply a composite of all of the included markdown
sources. The front matter of the including markdown source becomes the first element, and all other included are added
after this. The front matter of the including markdown becomes the representative front matter for the composite
markdown.
## Indexing Data
Indexing data should be provided in two forms:
1. The basic metadata index which includes topics, titles, and other basic info and logical path info. This view is used
to build menus for traversal and other simple views of topics as needed for direct presence check, or lookup.
2. A FTS index which includes a basic word index with stemming and other concerns pre-baked. This view is used as a
cache-friendly searchable index into the above metadata.

36
devdocs/docstructure/docsketch.md Normal file
View File

@ -0,0 +1,36 @@
# Doc System
This is a consolidation of all the doc system work thus far. This draft is meant to outline the basic features of the
doc system at a high level, but with suitable detail for an initial refactoring. In general this builds on existing work
in the doc system but with some adaptations for current needs, across CLI, apps, and reference material.
## Content Organization
All content loaded from any source is organized internally into a tree of sections by:
* Front Matter Topics
* Header Level
The source path of content does not matter. However, each unit of source material is considered its own section, with
zero or more additional subsections.
A root section is the container of all sections which are not homed under another section.
## Headings
In some cases, it is appropriate to consolidate individual docs into larger views. In order to facilitate this, all
sections within markdown structure are enumerated according to
- The front matter in the content source, specifically the topics assigned
- The heading structure within the doc
Thus, when the doc content is processed into the cohesive view needed by a user, all sections of all provided content
are cross-referenced and organized into sections.
The location of a document within the source filesystem or archive is not important. Topics
## Content Naming
## Content Searching

4
docsys/docs/docsys.md → devdocs/docstructure/docsys.md
View File

@ -7,10 +7,10 @@
- Only markdown files which contain front matter will be considered user-facing docs. - Only markdown files which contain front matter will be considered user-facing docs.
- Front matter will specify the namespace and/or names under which the included file should appear - Front matter will specify the namespace and/or names under which the included file should appear
- Front matter will include search meta and related topics - Front matter will include search meta and related topics
- Front matter will contain all the meta data that any client app needs to create a basic topic index or menu. - Front matter will contain all the metadata that any client app needs to create a basic topic index or menu.
- Front matter will specify whether or not to include the markdown in command line help. - Front matter will specify whether or not to include the markdown in command line help.
- Front matter will specify whether or not to include the markdown in web help. - Front matter will specify whether or not to include the markdown in web help.
- Front matter will specify the topic path for the included. - Front matter will specify the topic path for the included content.
### Internal APIs ### Internal APIs

0
docsys/docs/dynamic_endpoints.md → devdocs/dynamic_endpoints.md
View File

0
docsys/docs/inspirations.md → devdocs/inspirations.md
View File

75
devdocs/linearized/idealized.svg Normal file
View File

@ -0,0 +1,75 @@
<svg version="1.1" baseProfile="full" width="1038" height="114" viewbox="0 0 1038 114" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ev="http://www.w3.org/2001/xml-events" style="font-weight:bold; font-size:12pt; font-family:'Calibri', Helvetica, sans-serif;;stroke-width:3;stroke-linejoin:round;stroke-linecap:round">
<title >nomnoml</title>
<desc ># direction: right
#.op: fill=white visual=note direction=right
#.combined: fill=#EEEEFF visual=note direction=right
#.capture: fill=white visual=sender
#.value: fill=white visual=none
#.input: fill=white visual=receiver
//[&lt;op&gt; a]
//[&lt;capture&gt; a:username]
//[&lt;op&gt; b]
//[&lt;capture&gt; b:result]
[&lt;value&gt; cycle]
[&lt;value&gt; username]
[&lt;value&gt; result]
[cycle] -&gt; [op a]
[&lt;combined&gt;op a|
[&lt;input&gt; cycle]-&gt;[&lt;op&gt; a]
[&lt;op&gt;a]-&gt;[&lt;capture&gt;a:username]]
[op a] -&gt; [username]
[username] -&gt; [op b]
[&lt;combined&gt;op b|
[&lt;input&gt; username] -&gt; [&lt;op&gt; b]
[&lt;op&gt;b]-&gt;[&lt;capture&gt;b:result]]
[op b] -&gt; [result]</desc>
<path d="M77.5 57.5 L97.5 57.5 L117.5 57.5 L117.5 57.5 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M104.2 62.8 L110.8 57.5 L104.2 52.2 L117.5 57.5 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M424.5 57.5 L444.5 57.5 L464.5 57.5 L464.5 57.5 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M451.2 62.8 L457.8 57.5 L451.2 52.2 L464.5 57.5 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M556.5 57.5 L576.5 57.5 L596.5 57.5 L596.5 57.5 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M583.2 62.8 L589.8 57.5 L583.2 52.2 L596.5 57.5 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M912.5 57.5 L932.5 57.5 L952.5 57.5 L952.5 57.5 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M939.2 62.8 L945.8 57.5 L939.2 52.2 L952.5 57.5 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M117.5 13.5 L416.5 13.5 L424.5 21.5 L424.5 101.5 L117.5 101.5 L117.5 13.5 Z" style="stroke:#33322E;fill:#EEEEFF;stroke-dasharray:none;"></path>
<path d="M416.5 13.5 L416.5 21.5 L424.5 21.5" style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<text x="271" y="35" style="fill: #33322E;font-weight:normal;text-anchor: middle;">op a</text>
<path d="M117.5 44.5 L424.5 44.5" style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M194.5 73 L214.5 73 L234.5 73 L234.5 73 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M221.2 78.3 L227.8 73 L221.2 67.7 L234.5 73 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M260.5 73 L280.5 73 L300.5 73 L300.5 73 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M287.2 78.3 L293.8 73 L287.2 67.7 L300.5 73 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M122.5 57.5 L194.5 57.5 L194.5 88.5 L122.5 88.5 L130.5 73 Z" style="stroke:#33322E;fill:white;stroke-dasharray:none;"></path>
<text x="162.5" y="79" style="fill: #33322E;font-weight:normal;text-anchor: middle;">cycle</text>
<path d="M234.5 57.5 L252.5 57.5 L260.5 65.5 L260.5 88.5 L234.5 88.5 L234.5 57.5 Z" style="stroke:#33322E;fill:white;stroke-dasharray:none;"></path>
<path d="M252.5 57.5 L252.5 65.5 L260.5 65.5" style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<text x="247.5" y="79" style="fill: #33322E;font-weight:normal;text-anchor: middle;">a</text>
<path d="M300.5 57.5 L403.5 57.5 L411.5 73 L403.5 88.5 L300.5 88.5 Z" style="stroke:#33322E;fill:white;stroke-dasharray:none;"></path>
<text x="356" y="79" style="fill: #33322E;font-weight:normal;text-anchor: middle;">a:username</text>
<path d="M596.5 13.5 L904.5 13.5 L912.5 21.5 L912.5 101.5 L596.5 101.5 L596.5 13.5 Z" style="stroke:#33322E;fill:#EEEEFF;stroke-dasharray:none;"></path>
<path d="M904.5 13.5 L904.5 21.5 L912.5 21.5" style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<text x="754.5" y="35" style="fill: #33322E;font-weight:normal;text-anchor: middle;">op b</text>
<path d="M596.5 44.5 L912.5 44.5" style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M701.5 73 L721.5 73 L741.5 73 L741.5 73 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M728.2 78.3 L734.8 73 L728.2 67.7 L741.5 73 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M767.5 73 L787.5 73 L807.5 73 L807.5 73 " style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<path d="M794.2 78.3 L800.8 73 L794.2 67.7 L807.5 73 Z" style="stroke:#33322E;fill:#33322E;stroke-dasharray:none;"></path>
<path d="M601.5 57.5 L701.5 57.5 L701.5 88.5 L601.5 88.5 L609.5 73 Z" style="stroke:#33322E;fill:white;stroke-dasharray:none;"></path>
<text x="655.5" y="79" style="fill: #33322E;font-weight:normal;text-anchor: middle;">username</text>
<path d="M741.5 57.5 L759.5 57.5 L767.5 65.5 L767.5 88.5 L741.5 88.5 L741.5 57.5 Z" style="stroke:#33322E;fill:white;stroke-dasharray:none;"></path>
<path d="M759.5 57.5 L759.5 65.5 L767.5 65.5" style="stroke:#33322E;fill:none;stroke-dasharray:none;"></path>
<text x="754.5" y="79" style="fill: #33322E;font-weight:normal;text-anchor: middle;">b</text>
<path d="M807.5 57.5 L891.5 57.5 L899.5 73 L891.5 88.5 L807.5 88.5 Z" style="stroke:#33322E;fill:white;stroke-dasharray:none;"></path>
<text x="853.5" y="79" style="fill: #33322E;font-weight:normal;text-anchor: middle;">b:result</text>
<text x="45.5" y="64" style="fill: #33322E;font-weight:normal;text-anchor: middle;">cycle</text>
<text x="510.5" y="64" style="fill: #33322E;font-weight:normal;text-anchor: middle;">username</text>
<text x="989" y="64" style="fill: #33322E;font-weight:normal;text-anchor: middle;">result</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.9 KiB

196
devdocs/linearized/linearized.md Normal file
View File

@ -0,0 +1,196 @@
# Linearized Operations
NOTE: This is a sketch/work in progress and will not be suitable for earnest review until this notice is removed.
Thanks to Seb and Wei for helping this design along with their discussions along the way.
See https://github.com/nosqlbench/nosqlbench/issues/136
Presently, it is possible to stitch together rudimentary chained operations, as long as you already know how statement
sequences, bindings functions, and thread-local state work. This is a significant amount of knowledge to expect from a
user who simply wants to configure chained operations with internal dependencies.
The design changes needed to make this easy to express are non-trivial and cut across a few of the extant runtime
systems within nosqlbench. This design sketch will try to capture each of the requirements and approached sufficiently
for discussion and feedback.
# Sync and Async
## As it is: Sync vs Async
The current default mode (without `async=`) emulates a request-per-thread model, with operations being planned in a
deterministic sequence. In this mode, each thread dispatches operations from the sequence only after the previous one is
fully completed, even if there is no dependence between them. This is typical of many applications, even today, but not
all.
On the other end of the spectrum is the fully asynchronous dispatch mode enabled with the `async=` option. This uses a
completely different internal API to allow threads to juggle a number of operations. In contrast to the default mode,
the async mode dispatches operations eagerly as long as the user's selected concurrency level is not yet met. This means
that operations may overlap and also occur out of order with respect to the sequence.
Choosing between these modes is a hard choice that does not offer a uniform way of looking at operations. As well, it
also forces users to pick between two extremes of all request-per-thread or all asynchronous, which is becoming less
common in application designs, and at the very least does not rise to the level of expressivity of the toolchains that
most users have access to.
## As it should be: Async with Explicit Dependencies
* The user should be able to create explicit dependencies from one operation to another.
* Operations which are not dependent on other operations should be dispatched as soon as possible within the concurrency
limits of the workload.
* Operations with dependencies on other operations should only be dispatched if the upstream operations completed
successfully.
* Users should have clear expectations of how error handling will occur for individual operations as well
as chains of operations.
# Dependent Ops
We are using the phrase _dependent ops_ to capture the notions of data-flow dependency between ops (implying
linearization in ordering and isolation of input and output boundaries), successful execution, and data sharing within
an appropriate scope.
## As it is: Data Flow
Presently, you can store state within a thread local object map in order to share data between operations. This is using
the implied scope of "thread local" which works well with the "sequence per thread, request per thread" model. This
works because both the op sequence as well as the variable state used in binding functions are thread local.
However, it does not work well with the async mode, since there is no implied scope to tie the variable state to the op
sequence. There can be many operations within a thread operating on the same state even concurrently. This may appear to
function, but will create problems for users who are not aware of the limitation.
## As it should be: Data Flow
* Data flow between operations should be easily expressed with a standard configuration primitive which can work across
all driver types.
* The scope of data shared should be
The scope of a captured value should be clear to users
## As it is: Data Capture
Presently, the CQL driver has additional internal operators which allow for the capture of values. These decorator
behaviors allow for configured statements to do more than just dispatch an operation. However, they are not built upon
standard data capture and sharing operations which are implemented uniformly across driver types. This makes scope
management largely a matter of convention, which is ok for the first implementation (in the CQL driver) but not as a
building block for cross-driver behaviors.
# Injecting Operations
## As it is: Injecting Operations
Presently operations are derived from statement templates on a deterministic op sequence which is of a fixed length
known as the stride. This follows closely the pattern of assuming each operation comes from one distinct cycle and that
there is always a one-to-one relationship with cycles. This has carried some weight internally in how metrics for cycles
are derived, etc. There is presently no separate operational queue for statements except by modifying statements in the
existing sequence with side-effect binding assignment. It is difficult to reason about additional operations as
independent without decoupling these two into separate mechanisms.
## As it should be: Injecting Operations
## Seeding Context
# Diagrams
![idealized](idealized.svg)
## Op Flow
To track
Open concerns
- before: variable state was per-thread
- now: variable state is per opflow
- (opflow state is back-filled into thread local as the default implementation)
* gives scope for enumerating op flows, meaning you opflow 0... opflow (cycles/stride)
* 5 statements in sequence, stride=5,
- scoping for state
- implied data flow dependence vs explicit data flow dependence
- opflow retries vs op retries
discussion
```yaml
bindings:
yesterday: HashRange(0L,1234234L);
statements:
- s1-with-binding: select [userid*] from foobar.baz where day=23
- s2-with-binding: select [userid],[yesterday] from accounts where id={id} and timestamp>{yesterday}
- s3-with-dependency: select login_history from sessions where userid={[userid]}
- rogue-statement: select [yesterday] from ... <--- WARN USER because of explicit dependency below
- s4: select login_history from sessions where userid={[userid]} and timestamp>{yesterday}
- s5: select login_history from sessions where userid={[userid]} and timestamp>{[s2-with-binding/yesterday]}
```
## Dependency Indirection
## Error Handling and DataFlow Semantics
## Capture Syntax
Capturing of variables in statement templates will be signified with `[varname]`. This examples represents the simplest
case where the user just wants to capture a varaible. Thus the above is taken to mean:
- The scope of the captured variable is the OpFlow.
- The operation is required to succeed. Any other operation which depends on a `varname` value will be skipped and
counted as such.
- The captured type of `varname` is a single object, to be determined dynamically, with no type checking required.
- A field named `varname` is required to be present in the result set for the statement that included it.
- Exactly one value for `varname` is required to be present.
- Without other settings to relax sanity constraints, any other appearance of `[varname]` in another active statement
should yield a warning to the user.
All behavioral variations that diverge from the above will be signified within the capture syntax as a variation on the
above example.
## Inject Syntax
Similar to binding tokens used in statement templates like '{varname}', it is possible to inject captured variables into
statement templates with the `{[varname]}` syntax. This indicates that the user explicitly wants to pull a value
directly from the captured variable. It is necessary to indicate variable capture and variable injection distinctly from
each other, and this syntax supports that while remaining familiar to the bindings formats already supported.
The above syntax example represents the case where the user simply wants to refer to a variable of a given name. This is
the simplest case, and is taken to mean:
- The scope of the variable is not specified. The value may come from OpFlow, thread, global or any scope that is
available. By default, scopes should be consulted with the shortest-lived inner scopes first and widened only if
needed to find the variable.
- The variable must be defined in some available scope. By default, It is an error to refer to a variable for injection
that is not defined.
- The type of the variable is not checked on access. The type is presumed to be compatible with any assignments which
are made within whatever driver type is in use.
- The variable is assumed to be a single-valued type.
All behavioral variations that diverge from the above will be signified within the variable injection syntax as a
variation on the above syntax.
## Scenarios to Consider
basic scenario: user wants to capture each variable from one place
advanced scenarios:
- user wants to capture a named var from one or more places
- some ops may be required to complete successfully, others may not
- some ops may be required to produce a value
- some ops may be required to produce multiple values
* The carrier of op state should enable the following programmatic constructions:
* Metric measuring the service time of the op on failure
* Metric measuring the service time of the op on success
* Metric measuring the size of the op on success
* Hooks for transforming or acting upon the op or cycle before the op executes
* Hooks for transforming or acting upon the op or cycle after the op executes, regardless of result
* Additional modifiers on the op, as in transformers.
* All op contextual actions should be presented as a function on the op type
* Completion Stages that support the op API should come from built-in template implementations that already include
metrics options, logging support, etc.

185
devdocs/nbui/README.md Normal file
View File

@ -0,0 +1,185 @@
# NBUI
This is where the developer docs are kept for the NoSQLBench UI stack.
## Status
Presently (June/2020), NoSQLBench contains a web application and services layer, but it has not been
used for more than the documentation system. We are putting out request for comments and ideas and
for contributors who can build UIs in Vue.js around these ideas. This document is the first
consolidated view of how UIs work with NoSQLBench.
## NBUI Stack
The current _NBUI Stack_ consists of the NoSQLBench "nb" process, including the built-in web server,
The JAX-RS services which provide endpoints for accessing NoSQLBench functionality, and a set of
embedded web applications which are bundled within the nb application itself.
The docs for NoSQLBench are in-fact a Vue application. We use Nuxt as an framework/wrapper around
Vue to allow us to handle pre-rendering of the application for bundling. The docs application is
only an example of a Vue app in the NBUI stack, but it can be helpful for understanding how the
stack works.
The web apps are written in Vue.js as a separate developer workflow, but within the umbrella of the
NoSQLBench project. The documentation app lives in `src/main/node/docsys` under the docsys module.
It is a node and npm enabled Nuxt.js application. You can go to that directory and run it separately
under npm or yarn, which is what you would normally do when building any Vue/Nuxt application with
node js.
For bundling applications within the NoSQLBench distribution (jar or binary), Nuxt is used to
generate the "static site" form of the application. The result is packaged in
`src/main/resources/docsys-guidebook` and registered for sharing via the internal NoSQLBench static
content handler by way of the *DocsysDefaultAppPath* class, which is how all content sources are
found in NoSQLBench.
## NBUI Modes
Because the backing content for the docserver can come from a variety of sources, NBUI applications
can be run in different modes. Which mode you use depends on what you are doing, as explained below.
### Embedded Mode
This is the default mode that you get when you run `nosqlbench docserver`. In this mode, nosqlbench
starts up a Jetty container to serve content and JAX-RS services. One of the services is a static
content service which serves up the static copy of the generated UI application. This application is
static content on the server, but it includes a client-side application that further does dynamic
rendering and makes its own requests back to the server to fetch additional content or interact with
NoSQLBench service endpoints. This mode is illustrated by the following diagram:
![DocServer Embedded Mode](embedded_mode.png)
### Development Mode
This is the mode that UI developers will care about the most. In this mode, UI developers can run
Vue apps in the usual dev mode while accessing endpoints in a running nosqlbench instance. To use
this mode, you start nosqlbench with the `nosqlbench docserver` command, and then separately run the
local Nuxt app with `npm dev` which is configured to run nuxt in dev mode.
This causes the services to wire together as illustrated in this diagram:
![DocServer Dev Mode](dev_mode.png)
### Static Site Mode
This mode is used to host an NBUI app on a static site, where endpoint services are not available.
In this mode, content which would be returned from an endpoint path is statically exported into
files in the same path, such that a static content server with no dynamic page rendering can still
provide a snapshot of content to be used by the client-side applications.
The release pipeline creates this content with content export utilities, so that when new NoSQLBench
versions are released, the docs site is automatically updated with refresh of current content from
the source tree.
Client-side logic still runs (The client-side apps are always active), but the server simply hands
static content back from the file system:
![Static Site Mode](staticsite_mode.png)
## Interactive Or Not?
The three modes are useful and needed for different purposes:
- **Dev Mode** is needed for rapid development and prototyping of UIs and endpoint services. Dev
mode is the only mode where you can make code changes and immediately refresh to see the results
of your work. Dev mode supports full UI interactivity and endpoint services, so all application
features can be developed.
- **Embedded Mode** is for users of NoSQLBench who need to actually use the built UIs. Like Dev
mode, it supports full UI interactivity and endpoint services.
- **Static Mode** is for non-interactive use, and does not support endpoint service against an
instance of NoSQLBench, since it is not served from a NoSQLBench process that can also provide
these services.
Given that the static mode can't work with stateful services on a NoSQLBench server, a distinction
has to be made within an application about whether should support static mode. We'll put a
client-side check in where needed once we have other examples to work with. At the minimum, either
applications, or specific (interactive) application features will be disabled automatically if the
client-side application detects that endpoint services are not available. A future enhancement will
probably mean that the endpoint servers are fully configured and enabled by the client user as
needed.
### Developing with Nuxt
Using dev mode, you can build a vue.js application, configure it for being used as a _generated_ web
application, and then have it be hosted by NoSQLBench.
## Proposed Features
We propose these UI features to make using NoSQLBench easier:
### NBUI Wishlist
A user should be able to see all the NBUI apps from one place. They should be able to see the name
of each app, and some hover/title docs about what the app is used for. Users should be able to
switch to an app by clicking its name/menu entry. If a user has some unsaved work in one app that
they would lose by switching to another app, they should be prompted and given the choice to save or
discard their work first, or to cancel the action.
The NBUI Overview is not a highly functional app, but what it does is very important; It ties the
rest of the apps together within one cohesive view for the user. As such, it establishes a pattern
for the rest of the visual design of NBUI.
Sketch: A simple implementation of NBUI would be a frame holder with a menu of apps on the left. The
name of the current app could be the only app name showing by default. When the current app name is
clicked, it could unroll into a list of available app that the user could then pick between. Picking
another app would switch the currently active app within the overview frame and roll-up the app
names again to a single value. This basic design would leave a usable menu area below the app name
(and a subtle divider) so that the screen is still usable by each app, including the left. The left
panel could be toggled to be hidden, with a small corner visible to call it back.
### Scenario Runner
A user should be able to find all the named scenarios and select one to
run. They should be able to easily modify the parameters which the named scenario provides in
template variables. They should be able to run the configured scenario from the UI.
@phact (Sebastian) has already built one of these as a prototype. It would be a good starting point
for learning how web apps work in NoSQLBench. It will also need some updates to mesh well with
recent named scenarios features. This app needs to be tested and integrated into the main NBUI view.
Users should be able to run arbitrary commands in the scenario runner as well, just as they would on
the command line. This would allow them to use and test the rest of the UI features from a single
starting point.
### Scenario Status
An instance of a NoSQLBench process can run multiple scenarios, even concurrently. A user running
scenarios from NBUI will want to be able to see their overall status. This does not include metrics
status at this point, since the best view of this is largely provided already within the docker
metrics view.
Sketch: The scenario status app should list an info panel for each started or completed scenario. It
should contain the following details:
* The name of the scenario (All scenarios have a provided or auto-generated name)
* The commands which were used to start the scenario
* When it was started
* When it completed OR The current progress
* An ETA of completion
### VirtData Function Sandbox
A user should be able to find functions, chain them together, adjust their parameters within valid
ranges/values, and see example outputs over some input cycle range.
The preview of output values should be selectable between these types of visualizations:
- A histogram plot
- A list of values
Each output type is either numeric or non-numeric. In the degenerate case, numeric values should be
allowed to be converted to string form.
A user should be able to choose how they visualize the output: As either list of cycle to value
mappings, or a list of values, a summary of list values, or as a histogram. Additionally, numeric
output types should be plottable if possible.
### Workload Builder
Users should be able to build their workloads from the ground up. They should be able to take an
existing workload yaml as a starting point and tailor it as needed, and then save it.
This can take on the form of a syntax checking text editor, or as a node-based editor that allows
you to only add valid elements in the right place. In any case, the yaml form of the workload should
be accessible.

BIN
devdocs/nbui/dev_mode.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 128 KiB

50
devdocs/nbui/dev_mode.puml Normal file
View File

@ -0,0 +1,50 @@
@startuml
title NoSQLBench docserver in dev mode
participant "UserBrowser" as u
participant "Nuxt dev mode\n:3000 (HTTP)" as dev
participant "NoSQLBench\n:12345 (HTTP)" as nb
participant "Static Content\nService" as content
participant "Markdown Content\nService" as SPI
u -> dev : Load Page
activate dev
dev -> dev : render Vue app
u <- dev : <app resources>
deactivate dev
note over dev
Vue.js develpment occurs
in the Nuxt/Vue instance,
and supports dynamic layout
and reloading.
end note
u -> nb: Read Content Manifest
note over nb
Nuxt/Vue selects
this port for services
when Nuxt is in dev mode
on port 3000
end note
activate nb
nb -> SPI : List Content
activate SPI
SPI -> SPI: Discover and \nEnumerate
nb <- SPI : markdown\nmanifest
deactivate SPI
u <- nb: markdown\nmanifest
deactivate nb
u -> nb: Read Named Content
activate nb
nb -> SPI : Lookup
activate SPI
SPI -> SPI: Lookup
nb <- SPI : Named Content
deactivate SPI
u <- nb: Named Content
deactivate nb
@enduml

BIN
devdocs/nbui/embedded_mode.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 116 KiB

46
devdocs/nbui/embedded_mode.puml Normal file
View File

@ -0,0 +1,46 @@
@startuml
title NoSQLBench docserver in embedded mode
participant "UserBrowser" as u
participant "NoSQLBench\n:12345 (HTTP)" as nb
participant "Static Content\nService" as content
participant "Markdown Content\nService" as SPI
u -> nb : Load Page
activate nb
nb -> content : Read Generated App
activate content
nb <- content : static files
deactivate content
u <- nb : HTTP Content
deactivate nb
note over u
The client app initializes
and makes subsequent
calls to the origin server
for content and services
end note
u -> nb: Read Content Manifest
activate nb
nb -> SPI : List Content
activate SPI
SPI -> SPI: Discover and \nEnumerate
nb <- SPI : markdown\nmanifest
deactivate SPI
u <- nb: markdown\nmanifest
deactivate nb
u -> nb: Read Named Content
activate nb
nb -> SPI : Lookup
activate SPI
SPI -> SPI: Lookup
nb <- SPI : Named Content
deactivate SPI
u <- nb: Named Content
deactivate nb
... Subsequent calls are similar ...
@enduml

BIN
devdocs/nbui/staticsite_mode.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 114 KiB

50
devdocs/nbui/staticsite_mode.puml Normal file
View File

@ -0,0 +1,50 @@
@startuml
title NoSQLBench docserver in static mode
participant "UserBrowser" as u
participant "Generic Static\nContent Server\n(HTTP)" as nb
participant "Files" as content
u -> nb : Load Page
activate nb
nb -> content : "Read Generated App\n(static on disk)"
activate content
nb <- content : static files
deactivate content
u <- nb : HTTP Content
deactivate nb
note over u
The client app initializes
and makes subsequent
calls to the origin server
for content and services
[ which happen to be static
files in this case ]
end note
u -> nb: Read Content Manifest
activate nb
nb -> content : "Read Static Manifest\n(static on disk)"
activate content
note over content
To support this mode,
content which is normally
accessed via named endpoint
is statically included
as files at the same URI
path.
end note
nb <- content : markdown\nmanifest
deactivate content
u <- nb: Markdown Manifest
deactivate nb
... Remaining requests are similar ...
@enduml

44
devdocs/sketches/chain.dot Normal file
View File

@ -0,0 +1,44 @@
digraph {
// rankdir=LR;
node [shape = none]
cycles[label="read input"]
c_err[shape="none",label="[ERR,null]"]
c_ok[shape="none",label="[OK,cycle]"]
cycles -> c_err
cycles -> c_ok
bind_template[label="bind template"]
template_ok[label="[OK,template]"]
template_err[label="[ERR,template]"]
c_ok -> bind_template
bind_template -> template_err
bind_template -> template_ok
exec_cmd[label="execute command"]
command_err[label="[ERR,template]"]
command_ok[label="[OK,result]"]
template_ok -> exec_cmd
exec_cmd -> command_err
exec_cmd -> command_ok
verify_result[label="verify result"]
command_ok -> verify_result
result_invalid[label="[ERR,result]"]
result_ok[label="[OK,result]"]
verify_result -> result_invalid
verify_result -> result_ok
// <tr><td>Error<sub>none</sub></td></tr>
// ⁅⁆⟦⟧ ⟬⟭ ⟮⟯ ⟨⟩ ⁅⁆
// <td rowspan="*"><FONT POINT-SIZE="32">{</FONT></td>
// <td rowspan="*"><FONT POINT-SIZE="32">}</FONT></td>
}

80
devdocs/sketches/chain2.dot Normal file
View File

@ -0,0 +1,80 @@
digraph {
node [shape = none]
// rankdir=LR;
do_read_input[label="read input"]
c_err[shape="none",label="[ERR,null]"fontcolor=red]
c_ok[shape="none",label="[OK,cycle]",fontcolor=blue]
do_read_input -> c_err [color=red]
do_read_input -> c_ok [color=blue]
bind_template[label="bind template"]
template_ok[label="[OK,template]",fontcolor=blue]
template_err[label="[ERR,cycle]"fontcolor=red]
c_ok -> bind_template
bind_template -> template_err [color=red]
bind_template -> template_ok [color=blue]
exec_cmd[label="execute command"]
command_err[label="[ERR,template]"fontcolor=red]
command_ok[label="[OK,result]",fontcolor=blue]
template_ok -> exec_cmd
exec_cmd -> command_err [color=red]
exec_cmd -> command_ok [color=blue]
verify[label="verify result"]
command_ok -> verify
verify_err[label="[ERR,result]",fontcolor=red]
verify_ok[label="[OK,status]",fontcolor=blue]
verify -> verify_err [color=red]
verify -> verify_ok [color=blue]
clusterrank="local"
{
c_ok; command_ok; verify_ok
}
{
do_read_input; bind_template; exec_cmd; verify;
}
{
c_err; command_err; template_err; verify_err;
}
{
rank=sink;
errs[fontcolor=red];
ok[fontcolor=blue];
}
template_err -> errs [color=red]
verify_err -> errs [color=red]
c_err -> errs [color=red]
command_err -> errs [color=red]
// c_ok -> ok;
// template_ok -> ok;
// command_ok -> ok;
verify_ok -> ok [color=blue]
// <tr><td>Error<sub>none</sub></td></tr>
// ⁅⁆⟦⟧ ⟬⟭ ⟮⟯ ⟨⟩ ⁅⁆
// <td rowspan="*"><FONT POINT-SIZE="32">{</FONT></td>
// <td rowspan="*"><FONT POINT-SIZE="32">}</FONT></td>
}

27
devdocs/sketches/dockerstack.dot Normal file
View File

@ -0,0 +1,27 @@
digraph dockerstack {
rankdir=TB
node[shape=record]
// edge[fontsize=10]
nosqlbench[label="nosqlbench",shape="box"]
subgraph clusterfoo {
grafana[label="<3000>TCP/3000 (http)|grafana\ndocker", shape="record", rank=3]
graphite[label="{<9108>TCP/9108 (http)|<9109>TCP/9109 (graphite)}|graphite\ndocker", shape="record",
rank=2]
prometheus[label="<9090>TCP/9090 (promql)|prometheus\ndocker", shape="record", rank=3]
prometheus->graphite:9108
grafana->prometheus:9090
}
browser -> grafana:3000;
browser -> prometheus:9090;
nosqlbench->graphite:9109
{ rank=same; nosqlbench; browser; }
// { rank=same; graphite; grafana; }
}

BIN
devdocs/sketches/dockerstack.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 35 KiB

34
docsys/pom.xml
View File

@ -9,7 +9,7 @@
<parent> <parent>
<artifactId>mvn-defaults</artifactId> <artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId> <groupId>io.nosqlbench</groupId>
<version>3.12.83-SNAPSHOT</version> <version>3.12.127-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath> <relativePath>../mvn-defaults</relativePath>
</parent> </parent>
@ -18,7 +18,7 @@
<dependency> <dependency>
<groupId>io.nosqlbench</groupId> <groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId> <artifactId>nb-api</artifactId>
<version>3.12.83-SNAPSHOT</version> <version>3.12.127-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
@ -112,37 +112,9 @@
<dependency> <dependency>
<groupId>io.nosqlbench</groupId> <groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId> <artifactId>virtdata-api</artifactId>
<version>3.12.83-SNAPSHOT</version> <version>3.12.127-SNAPSHOT</version>
</dependency> </dependency>
</dependencies> </dependencies>
<profiles>
<profile>
<id>shade</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<groupId>org.apache.maven.plugins</groupId>
<configuration>
<transformers combine.children="append">
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>io.nosqlbench.docsys.core.DocServerApp</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
</transformers>
<minimizeJar>false</minimizeJar>
<finalName>${project.artifactId}</finalName>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project> </project>

8
docsys/src/main/java/io/nosqlbench/docsys/core/DocServerApp.java
View File

@ -1,12 +1,10 @@
package io.nosqlbench.docsys.core; package io.nosqlbench.docsys.core;
import org.slf4j.Logger; import org.apache.logging.log4j.LogManager;
import org.slf4j.LoggerFactory; import org.apache.logging.log4j.Logger;
import java.io.File; import java.io.File;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
@ -14,7 +12,7 @@ import java.util.Arrays;
public class DocServerApp { public class DocServerApp {
public final static String APPNAME_DOCSERVER = "docserver"; public final static String APPNAME_DOCSERVER = "docserver";
private static Logger logger = LoggerFactory.getLogger(DocServerApp.class); private static Logger logger = LogManager.getLogger(DocServerApp.class);
// static { // static {
// // defer to an extant logger context if it is there, otherwise // // defer to an extant logger context if it is there, otherwise

2
docsys/src/main/java/io/nosqlbench/docsys/core/DocServerStatusEndpoint.java
View File

@ -1,6 +1,6 @@
package io.nosqlbench.docsys.core; package io.nosqlbench.docsys.core;
import io.nosqlbench.nb.api.annotations.Service; import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.docsys.api.WebServiceObject; import io.nosqlbench.docsys.api.WebServiceObject;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;

2
docsys/src/main/java/io/nosqlbench/docsys/core/DocsysMarkdownEndpoint.java
View File

@ -1,6 +1,6 @@
package io.nosqlbench.docsys.core; package io.nosqlbench.docsys.core;
import io.nosqlbench.nb.api.annotations.Service; import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.docsys.api.DocsNameSpace; import io.nosqlbench.docsys.api.DocsNameSpace;
import io.nosqlbench.docsys.api.Docs; import io.nosqlbench.docsys.api.Docs;
import io.nosqlbench.docsys.api.DocsBinder; import io.nosqlbench.docsys.api.DocsBinder;

15
docsys/src/main/node/docsys/components/DocsMenu.vue
View File

@ -105,13 +105,13 @@
.v-list-item--disabled { .v-list-item--disabled {
color: #DDDDDD !important; color: #DDDDDD !important;
} }
.theme--light.v-list-item:not(.v-list-item--active):not(.v-list-item--disabled) {
div.theme--light.v-list-item:not(.v-list-item--active):not(.v-list-item--disabled) {
color: #DDDDDD !important;
}
a.theme--light.v-list-item:not(.v-list-item--active):not(.v-list-item--disabled) {
color: #DDDDDD !important; color: #DDDDDD !important;
} }
/*.v-list-item .theme--light.v-list-item {*/
/* color: #FFFFFF;*/
/*}*/
.nuxt-link-exact-active { .nuxt-link-exact-active {
/*color: #52c41a;*/ /*color: #52c41a;*/
@ -120,11 +120,6 @@
/*color: #000000;*/ /*color: #000000;*/
} }
/* .v-list-item--active */
/* .v-list-item--link */
/* a .nuxt-link-exact-active*/
/* a .nuxt-link-active */
.isactive { .isactive {
background-color: #7F828B; background-color: #7F828B;
color: #52c41a; color: #52c41a;

3
docsys/src/main/node/docsys/package-lock.json generated
View File

@ -1002,8 +1002,7 @@
"fsevents": { "fsevents": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz",
"integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA=="
"optional": true
}, },
"readdirp": { "readdirp": {
"version": "3.3.0", "version": "3.3.0",

2
docsys/src/main/resources/META-INF/services/javax.annotation.processing.Processor
View File

@ -1 +1 @@
io.nosqlbench.nb.api.processors.ServiceProcessor io.nosqlbench.nb.annotations.ServiceProcessor

97
docsys/src/main/resources/docs-for-docsys/docsys/design/topics.md Normal file
View File

@ -0,0 +1,97 @@
# Topics in Files
# DEPRECATED
# DEPRECATED
Since DocSys is meant to make it easy to add docs to a system
without doing much extra management, the ability to label and find
topics of interest must be built in.
That means that topics will be mostly provided by convention, where
the primary content simply _has_ a topic due to it's proper labeling.
With Markdown being the primary format for most or all of the primary
documentation, topics will be taken from specific places:
1. The directory structure containing the markdown files.
2. The files containing the markdown.
3. The headings within the markdown.
## Topic Structure
Of the three sources of information above, different facets of topics
are represented. Anything under a given directory name will automatically
be assigned to a category of that directory name. Further, if a directory
has a markdown file with the right content, it may specifically set
the category name. This is a future feature that will likely be called
_markdown annotations_ in DocSys. For now, the directory name itself is
simply used as a category name.
Each markdown file has a filename, but we aren't worried about that. The file
name itself is not significant to the content, and can be used to organize
the ordering of files in directory. For example, it is suitable to call
a markdown file `00_somecontent.md` since we aren't paying attention to the
file name. However, the file should have a canonical topic associated with it,
for when you want to find files that speak to specific topics as a whole.
These topics are called `File Topic`s, and are automatically given the topic
name of the first heading in the file.
## Headings as Topics
Within a markdown file, the level of heading determines how topics are nested together.
Any heading that is at a deeper level than the one that came immediately before it
is considered a sub-topic of that heading.
## Views of Topics
There are multiple ways to view the topics in the content provided:
### Topic List
As a list of topics, irrespective of the source and structure. In this form,
all the topics are presented as a flat list. This is suitable for indexing
topics, for example.
### File Topics
As a list of file topics. This view of topics provides all the topic names
that appear first for each file. This is suitable for finding or browsing
a set of round topics which are each round and cohesive within a single
markdown file. These topics *do* include the subtopics per file, but they
are provided as properties of the top level topics as given.
Because the file topics take the first heading as the name as the topic
name for the whole file, the first heading is not represented duplicitously
within the sub topics owned by a file topic. (It is simply promoted to the
file level)
### Header Topics
As a header topic tree. in this form, all of the topics that are found in any
markdown file are included in a hierarchic form. However, a file may contain
multiple topics at the same top level. This form does not include all the
file topics above as containers of their respective header topics. All headers
at the top-most level of topics are provided, even if there are multiple topics
that tie for the top-most level in a given file.
Since header structure within a markdown file is not generally strictly kept,
the header levels are *scrunched* before being presented. This means that,
starting from the top, each header that is a direct subtopic is hoisted up
to the next available level number such that there are no skipped levels. This
makes presentation of topics much easier for rendering.
## Naming
Within the view model, these elements are named
- topics.list
- topics.headers
- topics.files

4
docsys/src/main/resources/docsys-guidebook/200.html
View File

@ -1,9 +1,9 @@
<!doctype html> <!doctype html>
<html> <html>
<head> <head>
<title>NoSQLBench</title><meta data-n-head="1" charset="utf-8"><meta data-n-head="1" name="viewport" content="width=device-width,initial-scale=1"><meta data-n-head="1" data-hid="description" name="description" content="Docs App for NoSQLBench"><link data-n-head="1" rel="icon" type="image/x-icon" href="/favicon.ico"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900&display=swap"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css"><link rel="preload" href="/_nuxt/0feadcc9602758adfc03.js" as="script"><link rel="preload" href="/_nuxt/4be23a108097ec7c320d.js" as="script"><link rel="preload" href="/_nuxt/f28ec2c5c77be90ea577.js" as="script"><link rel="preload" href="/_nuxt/3179369742c1dbefdc94.js" as="script"> <title>NoSQLBench</title><meta data-n-head="1" charset="utf-8"><meta data-n-head="1" name="viewport" content="width=device-width,initial-scale=1"><meta data-n-head="1" data-hid="description" name="description" content="Docs App for NoSQLBench"><link data-n-head="1" rel="icon" type="image/x-icon" href="/favicon.ico"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900&display=swap"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css"><link rel="preload" href="/_nuxt/7c52b47a9d09e9501071.js" as="script"><link rel="preload" href="/_nuxt/2db30ff69d2689005afd.js" as="script"><link rel="preload" href="/_nuxt/c01ee420d65f0f86e261.js" as="script"><link rel="preload" href="/_nuxt/fa799dd3c7503934188d.js" as="script">
</head> </head>
<body> <body>
<div id="__nuxt"><style>#nuxt-loading{visibility:hidden;opacity:0;position:absolute;left:0;right:0;top:0;bottom:0;display:flex;justify-content:center;align-items:center;flex-direction:column;animation:nuxtLoadingIn 10s ease;-webkit-animation:nuxtLoadingIn 10s ease;animation-fill-mode:forwards;overflow:hidden}@keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}@-webkit-keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}#nuxt-loading>div,#nuxt-loading>div:after{border-radius:50%;width:5rem;height:5rem}#nuxt-loading>div{font-size:10px;position:relative;text-indent:-9999em;border:.5rem solid #f5f5f5;border-left:.5rem solid #fff;-webkit-transform:translateZ(0);-ms-transform:translateZ(0);transform:translateZ(0);-webkit-animation:nuxtLoading 1.1s infinite linear;animation:nuxtLoading 1.1s infinite linear}#nuxt-loading.error>div{border-left:.5rem solid #ff4500;animation-duration:5s}@-webkit-keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}</style><script>window.addEventListener("error",function(){var e=document.getElementById("nuxt-loading");e&&(e.className+=" error")})</script><div id="nuxt-loading" aria-live="polite" role="status"><div>Loading...</div></div></div> <div id="__nuxt"><style>#nuxt-loading{visibility:hidden;opacity:0;position:absolute;left:0;right:0;top:0;bottom:0;display:flex;justify-content:center;align-items:center;flex-direction:column;animation:nuxtLoadingIn 10s ease;-webkit-animation:nuxtLoadingIn 10s ease;animation-fill-mode:forwards;overflow:hidden}@keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}@-webkit-keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}#nuxt-loading>div,#nuxt-loading>div:after{border-radius:50%;width:5rem;height:5rem}#nuxt-loading>div{font-size:10px;position:relative;text-indent:-9999em;border:.5rem solid #f5f5f5;border-left:.5rem solid #fff;-webkit-transform:translateZ(0);-ms-transform:translateZ(0);transform:translateZ(0);-webkit-animation:nuxtLoading 1.1s infinite linear;animation:nuxtLoading 1.1s infinite linear}#nuxt-loading.error>div{border-left:.5rem solid #ff4500;animation-duration:5s}@-webkit-keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}</style><script>window.addEventListener("error",function(){var e=document.getElementById("nuxt-loading");e&&(e.className+=" error")})</script><div id="nuxt-loading" aria-live="polite" role="status"><div>Loading...</div></div></div>
<script type="text/javascript" src="/_nuxt/0feadcc9602758adfc03.js"></script><script type="text/javascript" src="/_nuxt/4be23a108097ec7c320d.js"></script><script type="text/javascript" src="/_nuxt/f28ec2c5c77be90ea577.js"></script><script type="text/javascript" src="/_nuxt/3179369742c1dbefdc94.js"></script></body> <script type="text/javascript" src="/_nuxt/7c52b47a9d09e9501071.js"></script><script type="text/javascript" src="/_nuxt/2db30ff69d2689005afd.js"></script><script type="text/javascript" src="/_nuxt/c01ee420d65f0f86e261.js"></script><script type="text/javascript" src="/_nuxt/fa799dd3c7503934188d.js"></script></body>
</html> </html>

1
docsys/src/main/resources/docsys-guidebook/_nuxt/06592ce2abe0b5804b1d.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/0feadcc9602758adfc03.js
View File

@ -1 +0,0 @@
!function(e){function r(data){for(var r,n,f=data[0],l=data[1],d=data[2],i=0,h=[];i<f.length;i++)n=f[i],Object.prototype.hasOwnProperty.call(o,n)&&o[n]&&h.push(o[n][0]),o[n]=0;for(r in l)Object.prototype.hasOwnProperty.call(l,r)&&(e[r]=l[r]);for(v&&v(data);h.length;)h.shift()();return c.push.apply(c,d||[]),t()}function t(){for(var e,i=0;i<c.length;i++){for(var r=c[i],t=!0,n=1;n<r.length;n++){var l=r[n];0!==o[l]&&(t=!1)}t&&(c.splice(i--,1),e=f(f.s=r[0]))}return e}var n={},o={11:0},c=[];function f(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,f),t.l=!0,t.exports}f.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var c,script=document.createElement("script");script.charset="utf-8",script.timeout=120,f.nc&&script.setAttribute("nonce",f.nc),script.src=function(e){return f.p+""+{0:"9830e47321545405404a",1:"7eb7a3a73dc5915a986c",2:"b8e81b2b492f6dd24df6",3:"17caa30bc5148c3b1167",6:"20e204977e331e222f04",7:"3075e9c63b07a6d88331",8:"8d3cf518e1e9d8352839",9:"ba36f1d15067f3126abb",10:"06592ce2abe0b5804b1d",13:"bdfd456d9b7407d77bf5"}[e]+".js"}(e);var l=new Error;c=function(r){script.onerror=script.onload=null,clearTimeout(d);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),c=r&&r.target&&r.target.src;l.message="Loading chunk "+e+" failed.\n("+n+": "+c+")",l.name="ChunkLoadError",l.type=n,l.request=c,t[1](l)}o[e]=void 0}};var d=setTimeout((function(){c({type:"timeout",target:script})}),12e4);script.onerror=script.onload=c,document.head.appendChild(script)}return Promise.all(r)},f.m=e,f.c=n,f.d=function(e,r,t){f.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},f.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},f.t=function(e,r){if(1&r&&(e=f(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(f.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)f.d(t,n,function(r){return e[r]}.bind(null,n));return t},f.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return f.d(r,"a",r),r},f.o=function(object,e){return Object.prototype.hasOwnProperty.call(object,e)},f.p="/_nuxt/",f.oe=function(e){throw console.error(e),e};var l=window.webpackJsonp=window.webpackJsonp||[],d=l.push.bind(l);l.push=r,l=l.slice();for(var i=0;i<l.length;i++)r(l[i]);var v=d;t()}([]);

1
docsys/src/main/resources/docsys-guidebook/_nuxt/20e204977e331e222f04.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/2db30ff69d2689005afd.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/3075e9c63b07a6d88331.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/3179369742c1dbefdc94.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/3484794277c7edf13895.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/407f8405125899b013f8.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/4be23a108097ec7c320d.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/51b41e8ba4499fa7c407.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/52a7cce22511daf04466.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/5dbfbf0f0448fbb70c73.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/74955e3a4ded093cd4a2.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/7c52b47a9d09e9501071.js Normal file
View File

@ -0,0 +1 @@
!function(e){function r(data){for(var r,n,c=data[0],l=data[1],d=data[2],i=0,h=[];i<c.length;i++)n=c[i],Object.prototype.hasOwnProperty.call(o,n)&&o[n]&&h.push(o[n][0]),o[n]=0;for(r in l)Object.prototype.hasOwnProperty.call(l,r)&&(e[r]=l[r]);for(v&&v(data);h.length;)h.shift()();return f.push.apply(f,d||[]),t()}function t(){for(var e,i=0;i<f.length;i++){for(var r=f[i],t=!0,n=1;n<r.length;n++){var l=r[n];0!==o[l]&&(t=!1)}t&&(f.splice(i--,1),e=c(c.s=r[0]))}return e}var n={},o={11:0},f=[];function c(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,c),t.l=!0,t.exports}c.e=function(e){var r=[],t=o[e];if(0!==t)if(t)r.push(t[2]);else{var n=new Promise((function(r,n){t=o[e]=[r,n]}));r.push(t[2]=n);var f,script=document.createElement("script");script.charset="utf-8",script.timeout=120,c.nc&&script.setAttribute("nonce",c.nc),script.src=function(e){return c.p+""+{0:"52a7cce22511daf04466",1:"407f8405125899b013f8",2:"74955e3a4ded093cd4a2",3:"a243bd228956b1ed8d5f",6:"8702cf295bbfbfa997dc",7:"c8bbbc322f8f1fe65a27",8:"5dbfbf0f0448fbb70c73",9:"3484794277c7edf13895",10:"51b41e8ba4499fa7c407",13:"e7c9fbb1e91c35f241b3"}[e]+".js"}(e);var l=new Error;f=function(r){script.onerror=script.onload=null,clearTimeout(d);var t=o[e];if(0!==t){if(t){var n=r&&("load"===r.type?"missing":r.type),f=r&&r.target&&r.target.src;l.message="Loading chunk "+e+" failed.\n("+n+": "+f+")",l.name="ChunkLoadError",l.type=n,l.request=f,t[1](l)}o[e]=void 0}};var d=setTimeout((function(){f({type:"timeout",target:script})}),12e4);script.onerror=script.onload=f,document.head.appendChild(script)}return Promise.all(r)},c.m=e,c.c=n,c.d=function(e,r,t){c.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},c.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},c.t=function(e,r){if(1&r&&(e=c(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(c.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)c.d(t,n,function(r){return e[r]}.bind(null,n));return t},c.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return c.d(r,"a",r),r},c.o=function(object,e){return Object.prototype.hasOwnProperty.call(object,e)},c.p="/_nuxt/",c.oe=function(e){throw console.error(e),e};var l=window.webpackJsonp=window.webpackJsonp||[],d=l.push.bind(l);l.push=r,l=l.slice();for(var i=0;i<l.length;i++)r(l[i]);var v=d;t()}([]);

1
docsys/src/main/resources/docsys-guidebook/_nuxt/7eb7a3a73dc5915a986c.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/8702cf295bbfbfa997dc.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/8d3cf518e1e9d8352839.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/9830e47321545405404a.js
View File

File diff suppressed because one or more lines are too long

2
docsys/src/main/resources/docsys-guidebook/_nuxt/17caa30bc5148c3b1167.js → docsys/src/main/resources/docsys-guidebook/_nuxt/a243bd228956b1ed8d5f.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/b8e81b2b492f6dd24df6.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/ba36f1d15067f3126abb.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/bdfd456d9b7407d77bf5.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/c01ee420d65f0f86e261.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/c8bbbc322f8f1fe65a27.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/e7c9fbb1e91c35f241b3.js Normal file
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/f28ec2c5c77be90ea577.js
View File

File diff suppressed because one or more lines are too long

1
docsys/src/main/resources/docsys-guidebook/_nuxt/fa799dd3c7503934188d.js Normal file
View File

File diff suppressed because one or more lines are too long

4
docsys/src/main/resources/docsys-guidebook/index.html
View File

@ -1,9 +1,9 @@
<!doctype html> <!doctype html>
<html> <html>
<head> <head>
<title>NoSQLBench</title><meta data-n-head="1" charset="utf-8"><meta data-n-head="1" name="viewport" content="width=device-width,initial-scale=1"><meta data-n-head="1" data-hid="description" name="description" content="Docs App for NoSQLBench"><link data-n-head="1" rel="icon" type="image/x-icon" href="/favicon.ico"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900&display=swap"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css"><link rel="preload" href="/_nuxt/0feadcc9602758adfc03.js" as="script"><link rel="preload" href="/_nuxt/4be23a108097ec7c320d.js" as="script"><link rel="preload" href="/_nuxt/f28ec2c5c77be90ea577.js" as="script"><link rel="preload" href="/_nuxt/3179369742c1dbefdc94.js" as="script"> <title>NoSQLBench</title><meta data-n-head="1" charset="utf-8"><meta data-n-head="1" name="viewport" content="width=device-width,initial-scale=1"><meta data-n-head="1" data-hid="description" name="description" content="Docs App for NoSQLBench"><link data-n-head="1" rel="icon" type="image/x-icon" href="/favicon.ico"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900&display=swap"><link data-n-head="1" rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mdi/font@latest/css/materialdesignicons.min.css"><link rel="preload" href="/_nuxt/7c52b47a9d09e9501071.js" as="script"><link rel="preload" href="/_nuxt/2db30ff69d2689005afd.js" as="script"><link rel="preload" href="/_nuxt/c01ee420d65f0f86e261.js" as="script"><link rel="preload" href="/_nuxt/fa799dd3c7503934188d.js" as="script">
</head> </head>
<body> <body>
<div id="__nuxt"><style>#nuxt-loading{visibility:hidden;opacity:0;position:absolute;left:0;right:0;top:0;bottom:0;display:flex;justify-content:center;align-items:center;flex-direction:column;animation:nuxtLoadingIn 10s ease;-webkit-animation:nuxtLoadingIn 10s ease;animation-fill-mode:forwards;overflow:hidden}@keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}@-webkit-keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}#nuxt-loading>div,#nuxt-loading>div:after{border-radius:50%;width:5rem;height:5rem}#nuxt-loading>div{font-size:10px;position:relative;text-indent:-9999em;border:.5rem solid #f5f5f5;border-left:.5rem solid #fff;-webkit-transform:translateZ(0);-ms-transform:translateZ(0);transform:translateZ(0);-webkit-animation:nuxtLoading 1.1s infinite linear;animation:nuxtLoading 1.1s infinite linear}#nuxt-loading.error>div{border-left:.5rem solid #ff4500;animation-duration:5s}@-webkit-keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}</style><script>window.addEventListener("error",function(){var e=document.getElementById("nuxt-loading");e&&(e.className+=" error")})</script><div id="nuxt-loading" aria-live="polite" role="status"><div>Loading...</div></div></div> <div id="__nuxt"><style>#nuxt-loading{visibility:hidden;opacity:0;position:absolute;left:0;right:0;top:0;bottom:0;display:flex;justify-content:center;align-items:center;flex-direction:column;animation:nuxtLoadingIn 10s ease;-webkit-animation:nuxtLoadingIn 10s ease;animation-fill-mode:forwards;overflow:hidden}@keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}@-webkit-keyframes nuxtLoadingIn{0%{visibility:hidden;opacity:0}20%{visibility:visible;opacity:0}100%{visibility:visible;opacity:1}}#nuxt-loading>div,#nuxt-loading>div:after{border-radius:50%;width:5rem;height:5rem}#nuxt-loading>div{font-size:10px;position:relative;text-indent:-9999em;border:.5rem solid #f5f5f5;border-left:.5rem solid #fff;-webkit-transform:translateZ(0);-ms-transform:translateZ(0);transform:translateZ(0);-webkit-animation:nuxtLoading 1.1s infinite linear;animation:nuxtLoading 1.1s infinite linear}#nuxt-loading.error>div{border-left:.5rem solid #ff4500;animation-duration:5s}@-webkit-keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes nuxtLoading{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}</style><script>window.addEventListener("error",function(){var e=document.getElementById("nuxt-loading");e&&(e.className+=" error")})</script><div id="nuxt-loading" aria-live="polite" role="status"><div>Loading...</div></div></div>
<script type="text/javascript" src="/_nuxt/0feadcc9602758adfc03.js"></script><script type="text/javascript" src="/_nuxt/4be23a108097ec7c320d.js"></script><script type="text/javascript" src="/_nuxt/f28ec2c5c77be90ea577.js"></script><script type="text/javascript" src="/_nuxt/3179369742c1dbefdc94.js"></script></body> <script type="text/javascript" src="/_nuxt/7c52b47a9d09e9501071.js"></script><script type="text/javascript" src="/_nuxt/2db30ff69d2689005afd.js"></script><script type="text/javascript" src="/_nuxt/c01ee420d65f0f86e261.js"></script><script type="text/javascript" src="/_nuxt/fa799dd3c7503934188d.js"></script></body>
</html> </html>

2
docsys/src/test/java/io/nosqlbench/docsys/core/DocsysMarkdownEndpointTest.java → docsys/src/test/java/io/nosqlbench/docsys/core/DocsysMarkdownLoaderEndpointTest.java
View File

@ -2,7 +2,7 @@ package io.nosqlbench.docsys.core;
import org.junit.Test; import org.junit.Test;
public class DocsysMarkdownEndpointTest { public class DocsysMarkdownLoaderEndpointTest {
@Test @Test
public void testDocLoader() { public void testDocLoader() {

1
docsys/src/test/resources/testsite1/basic2.mvel
View File

@ -1 +0,0 @@
This is an example MVEL file @{ "testing" }

5
docsys/src/test/resources/testsite1/index.md
View File

@ -1,5 +0,0 @@
<HTML>
<HEAD></HEAD>
{{ files }}
<BODY></BODY>
</HTML>

5
docsys/src/test/resources/testsite1/json_matter.md
View File

@ -1,5 +0,0 @@
{
"title": "Example Title (JSON)"
}
## Basic Markdown File

613
docsys/src/test/resources/testsite1/standard_yaml.md
View File

@ -1,613 +0,0 @@
---
title: YAML Config
weight: 33
menu:
main:
parent: User Guide
identifier: configfiles
weight: 15
---
A standard YAML configuration format is
provided that makes it easy to use for any activity that requires statements,
tags, parameters and data bindings. In practice, any useful activity types have
needed these. This section describes the standard YAML format and how to use it.
A valid config file for an activity consists of statements, parameters for those
statements, bindings for the data to use with those statements, and tags for
selecting statements for an activity. In essence, the config format is *all about
configuring statements*. Every other element in the config format is in some way
modifying or otherwise helping create statements to be used in an activity.
## Statements
Statements are the single most important part of a YAML config:
```yaml
# a single statement
statement: a single statement body
```
This is a valid YAML in and of itself. It may not be valid for a particular
activity type, but that is up to each activity type to decide. That is because
the contents of the YAML will matter to each activity type in a different way.
Thus, each activity type determines what a statement means, and how it will be
used. The format is merely concerned with structure and very general semantics.
In the example above, the statement is automatically named according to its
location within the YAML document.
## Bindings
Procedural data generation is built-in to nosqlbench. Bindings are now
supported as a core concept. You can add a bindings section like this:
```yaml
bindings:
alpha: Identity()
beta: NumberNameToString()
gamma: Combinations('0-9A-F;0-9;A-Z;_;p;r;o;')
delta: WeightedStrings('one:1;six:6;three:3;')
```
Notice that bindings are represented as a map. The bindings above *mostly* match
up with the named anchors in the statement list above. There is one extra
binding, but this is ok. However, if the statement included named anchors for which no
binding was defined, an error would occur.
```
This is important for activity type designers to observe: When statement bindings
are paired with statements, a binding should not be used directly unless it is
paired with a matching anchor name in the statement. This allows activity and
scenario designers to keep a library of data binding recipes in their
configurations without incurring the cost of extraneous binding evaluations.
```
Still, the above statement block is a valid config file. It may or may not be
valid for a given activity type. The 'stdout' activity will synthesize a
statement template from the provided bindings if needed, so this is valid:
```text
[test]$ cat > stdout-test.yaml
bindings:
alpha: Identity()
beta: NumberNameToString()
gamma: Combinations('0-9A-F;0-9;A-Z;_;p;r;o;')
delta: WeightedStrings('one:1;six:6;three:3;')
# EOF (control-D in your terminal)
[test]$ ./eb run driver=stdout yaml=stdout-test cycles=10
0,zero,00A_pro,six
1,one,00B_pro,six
2,two,00C_pro,three
3,three,00D_pro,three
4,four,00E_pro,six
5,five,00F_pro,six
6,six,00G_pro,six
7,seven,00H_pro,six
8,eight,00I_pro,six
9,nine,00J_pro,six
```
If you combine the statement section and the bindings sections above into one
activity yaml, you get a slightly different result, as the bindings apply
to the statements that are provided, rather than creating a default statement
for the bindings. See the example below:
```text
[test]$ cat > stdout-test.yaml
statements:
- |
This is a statement, and the file format doesn't
know how statements will be used!
- |
submit job {alpha} on queue {beta} with options {gamma};
bindings:
alpha: Identity()
beta: NumberNameToString()
gamma: Combinations('0-9A-F;0-9;A-Z;_;p;r;o;')
delta: WeightedStrings('one:1;six:6;three:3;')
# EOF (control-D in your terminal)
[test]$ ./eb run driver=stdout yaml=stdout-test cycles=10
This is a statement, and the file format doesn't
know how statements will be used!
submit job 1 on queue one with options 00B_pro;
This is a statement, and the file format doesn't
know how statements will be used!
submit job 3 on queue three with options 00D_pro;
This is a statement, and the file format doesn't
know how statements will be used!
submit job 5 on queue five with options 00F_pro;
This is a statement, and the file format doesn't
know how statements will be used!
submit job 7 on queue seven with options 00H_pro;
This is a statement, and the file format doesn't
know how statements will be used!
submit job 9 on queue nine with options 00J_pro;
```
There are a few things to notice here. First, the statements that are executed
are automatically alternated between. If you had 10 different statements listed,
they would all get their turn with 10 cycles. Since there were two, each was run
5 times.
Also, the statement that had named anchors acted as a template, whereas the
other one was evaluated just as it was. In fact, they were both treated as
templates, but one of the had no anchors.
On more minor but important detail is that the fourth binding *delta* was not
referenced directly in the statements. Since the statements did not pair up an
anchor with this binding name, it was not used. No values were generated for it.
This is how activities are expected to work when they are implemented correctly.
This means that the bindings themselves are templates for data generation, only
to be used when necessary.
## Params
As with the bindings, a params section can be added at the same level, setting
additional parameters to be used with statements. Again, this is an example of
modifying or otherwise creating a specific type of statement, but always in a
way specific to the activity type. Params can be thought of as statement
properties. As such, params don't really do much on their own, although they
have the same basic map syntax as bindings:
```yaml
params:
ratio: 1
```
As with statements, it is up to each activity type to interpret params in a
useful way.
## Tags
Tags are used to mark and filter groups of statements for controlling which ones
get used in a given scenario:
```yaml
tags:
name: foxtrot
unit: bravo
```
### Tag Filtering
The tag filters provide a flexible set of conventions for filtering tagged statements.
Tag filters are usually provided as an activity parameter when an activity is launched.
The rules for tag filtering are:
1. If no tag filter is specified, then the statement matches.
2. A tag name predicate like `tags=name` asserts the presence of a specific
tag name, regardless of its value.
3. A tag value predicate like `tags=name:foxtrot` asserts the presence of
a specific tag name and a specific value for it.
4. A tag pattern predicate like `tags=name:'fox.*'` asserts the presence of a
specific tag name and a value that matches the provided regular expression.
5. Multiple tag predicates may be specified as in `tags=name:'fox.*',unit:bravo`
6. Tag predicates are joined by *and* when more than one is provided -- If any predicate
fails to match a tagged element, then the whole tag filtering expression fails
to match.
A demonstration...
```text
[test]$ cat > stdout-test.yaml
tags:
name: foxtrot
unit: bravo
statements:
- "I'm alive!\n"
# EOF (control-D in your terminal)
# no tag filter matches any
[test]$ ./eb run driver=stdout yaml=stdout-test
I'm alive!
# tag name assertion matches
[test]$ ./eb run driver=stdout yaml=stdout-test tags=name
I'm alive!
# tag name assertion does not match
[test]$ ./eb run driver=stdout yaml=stdout-test tags=name2
02:25:28.158 [scenarios:001] ERROR i.e.activities.stdout.StdoutActivity - Unable to create a stdout statement if you have no active statements or bindings configured.
# tag value assertion does not match
[test]$ ./eb run driver=stdout yaml=stdout-test tags=name:bravo
02:25:42.584 [scenarios:001] ERROR i.e.activities.stdout.StdoutActivity - Unable to create a stdout statement if you have no active statements or bindings configured.
# tag value assertion matches
[test]$ ./eb run driver=stdout yaml=stdout-test tags=name:foxtrot
I'm alive!
# tag pattern assertion matches
[test]$ ./eb run driver=stdout yaml=stdout-test tags=name:'fox.*'
I'm alive!
# tag pattern assertion does not match
[test]$ ./eb run driver=stdout yaml=stdout-test tags=name:'tango.*'
02:26:05.149 [scenarios:001] ERROR i.e.activities.stdout.StdoutActivity - Unable to create a stdout statement if you have no active statements or bindings configured.
# compound tag predicate matches every assertion
[test]$ ./eb run driver=stdout yaml=stdout-test tags='name=fox.*',unit=bravo
I'm alive!
# compound tag predicate does not fully match
[test]$ ./eb run driver=stdout yaml=stdout-test tags='name=fox.*',unit=delta
11:02:53.490 [scenarios:001] ERROR i.e.activities.stdout.StdoutActivity - Unable to create a stdout statement if you have no active statements or bindings configured.
```
## Blocks
All the basic primitives described above (names, statements, bindings, params,
tags) can be used to describe and parameterize a set of statements in a yaml
document. In some scenarios, however, you may need to structure your statements
in a more sophisticated way. You might want to do this if you have a set of
common statement forms or parameters that need to apply to many statements, or
perhaps if you have several *different* groups of statements that need to be
configured independently.
This is where blocks become useful:
```text
[test]$ cat > stdout-test.yaml
bindings:
alpha: Identity()
beta: Combinations('u;n;u;s;e;d;')
blocks:
- statements:
- "{alpha},{beta}\n"
bindings:
beta: Combinations('b;l;o;c;k;1;-;COMBINATIONS;')
- statements:
- "{alpha},{beta}\n"
bindings:
beta: Combinations('b;l;o;c;k;2;-;COMBINATIONS;')
# EOF (control-D in your terminal)
[test]$ ./eb run driver=stdout yaml=stdout-test cycles=10
0,block1-C
1,block2-O
2,block1-M
3,block2-B
4,block1-I
5,block2-N
6,block1-A
7,block2-T
8,block1-I
9,block2-O
```
This shows a couple of important features of blocks. All blocks inherit defaults
for bindings, params, and tags from the root document level. Any of these values
that are defined at the base document level apply to all blocks contained in
that document, unless specifically overridden within a given block.
## More Statements
### Statement Delimiting
Sometimes, you want to specify the text of a statement in different ways. Since
statements are strings, the simplest way for small statements is in double
quotes. If you need to express a much longer statement with special characters
an newlines, then you can use YAML's literal block notation (signaled by the '|'
character) to do so:
```yaml
statements:
- |
This is a statement, and the file format doesn't
know how statements will be used!
- |
submit job {alpha} on queue {beta} with options {gamma};
```
Notice that the block starts on the following line after the pipe symbol. This
is a very popular form in practice because it treats the whole block exactly as
it is shown, except for the initial indentations, which are removed.
Statements in this format can be raw statements, statement templates, or
anything that is appropriate for the specific activity type they are being used
with. Generally, the statements should be thought of as a statement form that
you want to use in your activity -- something that has place holders for data
bindings. These place holders are called *named anchors*. The second line above
is an example of a statement template, with anchors that can be replaced by data
for each cycle of an activity.
There is a variety of ways to represent block statements, with folding, without,
with the newline removed, with it retained, with trailing newlines trimmed or
not, and so forth. For a more comprehensive guide on the YAML conventions
regarding multi-line blocks, see [YAML Spec 1.2, Chapter 8, Block
Styles](http://www.yaml.org/spec/1.2/spec.html#Block)
### Statement Sequences
To provide a degree of flexibility to the user for statement definitions,
multiple statements may be provided together as a sequence.
```yaml
# a list of statements
statements:
- "This a statement."
- "The file format doesn't know how statements will be used."
- "submit job {job} on queue {queue} with options {options};"
# an ordered map of statements by name
statements:
name1: statement one
name2: "statement two"
```
In the first form, the names are provided automatically by the YAML loader. In
the second form, they are specified as ordered map keys.
### Statement Properties
You can also configure individual statements with named properties, using the
**statement properties** form:
```yaml
# a list of statements with properties
statements:
- name: name1
stmt: statement one
- name: name2
stmt: statement two
```
This is the most flexible configuration format at the statement level. It is
also the most verbose. Because this format names each property of the statement,
it allows for other properties to be defined at this level as well. This
includes all of the previously described configuration elements: `name`,
`bindings`, `params`, `tags`, and additionally `stmt`. A detailed example
follows:
```yaml
statements:
- name: foostmt
stmt: "{alpha},{beta}\n"
bindings:
beta: Combinations('COMBINATIONS;')
params:
parm1: pvalue1
tags:
tag1: tvalue1
freeparam3: a value, as if it were assigned under the params block.
```
In this case, the values for `bindings`, `params`, and `tags` take precedence,
overriding those set by the enclosing block or document or activity when the
names match. Parameters called **free parameters** are allowed here, such as
`freeparam3`. These are simply values that get assigned to the params map once
all other processing has completed.
It is possible to mix the **`<name>: <statement>`** form as above in the
example for mapping statement by name, so long as some specific rules are
followed. An example, which is equivalent to the above:
```yaml
statements:
- foostmt: "{alpha},{beta}\n"
parm1: pvalue1
bindings:
beta: Combinations('COMBINATIONS;')
tags:
tag1: tvalue1
```
The rules:
1. You must avoid using both the name property and the initial
**`<name>: <statement>`** together. Doing so will cause an error to be thrown.
2. Do not use the **`<name>: <statement>`** form in combination with a
**`stmt: <statement>`** property. It is not possible to detect if this occurs.
Use caution if you choose to mix these forms.
As explained above, `parm1: pvalue1` is a *free parameter*, and is simply
short-hand for setting values in the params map for the statement.
### Per-Statement Format
It is indeed possible to use any of the three statement formats within
each entry of a statement sequence:
```yaml
statements:
- first statement body
- second: second statement body
- name: statement3
stmt: third statement body
- forth: fourth statement body
freeparam1: freeparamvalue1
tags:
type: preload
```
Specifically, the first statement is a simple statement body, the second is a
named statement (via free param `<name>: statement` form), the third is a
statement config map, and the fourth is a combination of the previous two.
The above is valid nosqlbench YAML, although a reader would need
to know about the rules explained above in order to really make sense of it. For
most cases, it is best to follow one format convention, but there is flexibility
for overrides and naming when you need it.
## Multi-Docs
The YAML spec allows for multiple yaml documents to be concatenated in the
same file with a separator:
```yaml
---
```
This offers an additional convenience when configuring activities. If you want
to parameterize or tag some a set of statements with their own bindings, params,
or tags, but alongside another set of uniquely configured statements, you need
only put them in separate logical documents, separated by a triple-dash.
For example:
```text
[test]$ cat > stdout-test.yaml
bindings:
docval: WeightedStrings('doc1.1:1;doc1.2:2;')
statements:
- "doc1.form1 {docval}\n"
- "doc1.form2 {docval}\n"
---
bindings:
numname: NumberNameToString()
statements:
- "doc2.number {numname}\n"
# EOF (control-D in your terminal)
[test]$ ./eb run driver=stdout yaml=stdout-test cycles=10
doc1.form1 doc1.1
doc1.form2 doc1.2
doc2.number two
doc1.form1 doc1.2
doc1.form2 doc1.1
doc2.number five
doc1.form1 doc1.2
doc1.form2 doc1.2
doc2.number eight
doc1.form1 doc1.1
```
## Template Params
All YAML formats support a parameter macro format that applies before
YAML processing starts. It is a basic macro facility that allows named
anchors to be placed in the document as a whole:
```text
<<varname:defaultval>>
```
In this example, the name of the parameter is <code>varname</code>. It is given a default
value of <code>defaultval</code>. If an activity parameter named *varname* is provided, as
in <code>varname=barbaz</code>, then this whole expression including the double angle
brackets will be replaced with <code>barbaz</code>. If none is provided then the default
value. For example:
```text
[test]$ cat > stdout-test.yaml
statements:
- "<<linetoprint:MISSING>>\n"
# EOF (control-D in your terminal)
[test]$ ./eb run driver=stdout yaml=stdout-test cycles=1
MISSING
[test]$ ./eb run driver=stdout yaml=stdout-test cycles=1 linetoprint="THIS IS IT"
THIS IS IT
```
If an empty value is desired by default, then simply use an empty string in your template,
like `<<varname:>>`.
## Naming Things
Docs, Blocks, and Statements can all have names:
```yaml
name: doc1
blocks:
- name: block1
statements:
- stmt1: statement1
- name: st2
stmt: statement2
---
name: doc2
...
```
This provides a layered naming scheme for the statements themselves. It is not
usually important to name things except for documentation or metric naming
purposes.
### Automatic Naming
If no names are provided, then names are automatically created for blocks and
statements. Statements assigned at the document level are assigned to
"block0". All other statements are named with the format `doc#--block#--stmt#`.
For example, the full name of statement1 above would be `doc1--block1--stmt1`.
## Diagnostics
This section describes errors that you might see if you have a YAML loading issue, and what
you can do to fix them.
### Undefined Name-Statement Tuple
This exception is thrown when the statement body is not found in a statement definition
in any of the supported formats. For example, the following block will cause an error:
statements:
- name: statement-foo
params:
aparam: avalue
This is because `name` and `params` are reserved property names -- removed from the list of name-value
pairs before free parameters are read. If the statement is not defined before free parameters
are read, then the first free parameter is taken as the name and statement in `name: statement` form.
To correct this error, supply a statement property in the map, or simply replace the `name: statement-foo` entry
with a `statement-foo: statement body` at the top of the map:
Either of these will work:
statements:
- name: statement-foo
stmt: statement body
params:
aparam: avalue
statements:
- statement-foo: statement body
params:
aparam: avalue
In both cases, it is clear to the loader where the statement body should come from, and what (if any) explicit
naming should occur.
### Redefined Name-Statement Tuple
This exception is thrown when the statement name is defined in multiple ways. This is an explicit exception
to avoid possible ambiguity about which value the user intended. For example, the following statements
definition will cause an error:
statements:
- name: name1
name2: statement body
This is an error because the statement is not defined before free parameters are read, and the `name: statement`
form includes a second definition for the statement name. In order to correct this, simply remove the separate
`name` entry, or use the `stmt` property to explicitly set the statement body. Either of these will work:
statements:
- name2: statement body
statements:
- name: name1
stmt: statement body
In both cases, there is only one name defined for the statement according to the supported formats.
### YAML Parsing Error
This exception is thrown when the YAML format is not recognizable by the YAML parser. If you are not
working from examples that are known to load cleanly, then please review your document for correctness
according to the [YAML Specification]().
If you are sure that the YAML should load, then please [submit a bug report](https://github.com/nosqlbench/nosqlbench/issues/new?labels=bug)
with details on the type of YAML file you are trying to load.
### YAML Construction Error
This exception is thrown when the YAML was loaded, but the configuration object was not able to be constructed
from the in-memory YAML document. If this error occurs, it may be a bug in the YAML loader implementation.
Please [submit a bug report](https://github.com/nosqlbench/nosqlbench/issues/new?labels=bug) with details
on the type of YAML file you are trying to load.

9
docsys/src/test/resources/testsite1/toml_matter.md
View File

@ -1,9 +0,0 @@
+++
title: Example Title (TOML)
+++
# Header1
A Synopsis
## Header2ile

9
docsys/src/test/resources/testsite1/yaml_matter.md
View File

@ -1,9 +0,0 @@
---
title: Example Title (YAML)
---
# Header1
A Synopsis
## Header2

1
docsys/src/test/resources/testsite2/advanced/advanced_topic1.md
View File

@ -1 +0,0 @@
## This is just a header for advanced topic 1

155
driver-cql-shaded/pom.xml Normal file
View File

@ -0,0 +1,155 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>3.12.127-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<artifactId>driver-cql-shaded</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>
A Shaded CQL ActivityType driver for http://nosqlbench.io/
</description>
<dependencies>
<!-- core dependencies -->
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>3.12.127-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.datastax.dse</groupId>
<artifactId>dse-java-driver-core</artifactId>
</dependency>
<dependency>
<groupId>com.datastax.dse</groupId>
<artifactId>dse-java-driver-extras</artifactId>
</dependency>
<dependency>
<groupId>com.datastax.dse</groupId>
<artifactId>dse-java-driver-mapping</artifactId>
</dependency>
<!-- For CQL compression option -->
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
</dependency>
<!-- For CQL compression option -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</dependency>
<!-- <dependency>-->
<!-- <groupId>io.netty</groupId>-->
<!-- <artifactId>netty-transport-native-epoll</artifactId>-->
<!-- <version>4.1.36.Final</version>-->
<!-- <classifier>linux-x86_64</classifier>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.yaml</groupId>-->
<!-- <artifactId>snakeyaml</artifactId>-->
<!-- <version>1.23</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.apache.commons</groupId>-->
<!-- <artifactId>commons-lang3</artifactId>-->
<!-- <version>3.7</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <version>1.7.25</version>-->
<!-- </dependency>-->
<!-- test only scope -->
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals><goal>shade</goal></goals>
</execution>
</executions>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<createSourcesJar>true</createSourcesJar>
<!-- <shadedArtifactAttached>true</shadedArtifactAttached>-->
<!-- <shadedClassifierName>shaded</shadedClassifierName>-->
<relocations>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>com.datastax.internal.com_google_common</shadedPattern>
</relocation>
<!-- <relocation>-->
<!-- <pattern>com.datastax</pattern>-->
<!-- <shadedPattern>dse19.com.datastax</shadedPattern>-->
<!-- </relocation>-->
<!-- <relocation>-->
<!-- <pattern>io.netty</pattern>-->
<!-- <shadedPattern>dse19.io.netty</shadedPattern>-->
<!-- </relocation>-->
</relocations>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
</artifactSet>
<transformers combine.children="append">
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>io.nosqlbench.engine.cli.NBCLI</mainClass>
</transformer>
</transformers>
<!-- <finalName>${project.artifactId}</finalName>-->
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</plugin>
</plugins>
</build>
</project>

0
activitytype-cql/src/main/java/com/datastax/driver/core/M3PTokenFilter.java → driver-cql-shaded/src/main/java/com/datastax/driver/core/M3PTokenFilter.java
View File

0
activitytype-cql/src/main/java/com/datastax/driver/core/TokenRangeStmtFilter.java → driver-cql-shaded/src/main/java/com/datastax/driver/core/TokenRangeStmtFilter.java
View File

0
activitytype-cql/src/main/java/com/datastax/driver/core/TokenRangeUtil.java → driver-cql-shaded/src/main/java/com/datastax/driver/core/TokenRangeUtil.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/api/ErrorResponse.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/api/ErrorResponse.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/api/ResultSetCycleOperator.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/api/ResultSetCycleOperator.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/api/RowCycleOperator.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/api/RowCycleOperator.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/api/StatementFilter.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/api/StatementFilter.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/api/VerifyApplied.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/api/VerifyApplied.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/CQLUserTypeNames.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/CQLUserTypeNames.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTCodecClasses.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTCodecClasses.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTCodecInjector.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTCodecInjector.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTJavaType.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTJavaType.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTTransformCodec.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UDTTransformCodec.java
View File

0
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UserCodecProvider.java → driver-cql-shaded/src/main/java/io/nosqlbench/activitytype/cql/codecsupport/UserCodecProvider.java
View File

Some files were not shown because too many files have changed in this diff Show More