IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into nosqlbench-948-testcontainers

Browse Source
This commit is contained in:
Jonathan Shook 2023-02-02 17:02:10 -06:00
commit cb073b0abd
787 changed files with 1825 additions and 132476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/build.yml vendored
View File

@ -5,9 +5,9 @@ name: build
on:
push:
paths:
- "!RELEASE_NOTES.md"
- "!PREVIEW_NOTES.md"
tags:
- "![0-9]+.[0-9]+.[0-9]+-preview"
- "![0-9]+.[0-9]+.[0-9]+-release"
pull_request:
jobs:
@ -42,15 +42,15 @@ jobs:
- name: Running tests
run: mvn -B test
- name: Collecting reports
run: tar -cvf codecov-report.tar target/coverage-report/**/*
# - name: Collecting reports
# run: tar -cvf codecov-report.tar target/coverage-report/**/*
- name: Uploading test coverage
uses: actions/upload-artifact@v3
with:
node-version: '16'
name: codecov-report
path: codecov-report.tar
# - name: Uploading test coverage
# uses: actions/upload-artifact@v3
# with:
# node-version: '16'
# name: codecov-report
# path: codecov-report.tar
- name: Collecting logfiles
if: success() || failure()
@ -76,7 +76,7 @@ jobs:
name: exported-docs
path: exported_docs.zip
docs:
builddocs:
needs: build
runs-on: ubuntu-20.04
steps:

4
.github/workflows/codeql-analysis.yml vendored
View File

@ -14,6 +14,10 @@ name: "CodeQL"
on:
push:
branches: [ main ]
tags:
- "![0-9]+.[0-9]+.[0-9]+-preview"
- "![0-9]+.[0-9]+.[0-9]+-release"
pull_request:
# The branches below must be a subset of the branches above
branches: [ main ]

231
.github/workflows/preview.yml vendored
View File

@ -1,9 +1,12 @@
name: preview
# This workflow should run for any push which is a preview build.
# Preview builds are indicated with changes to PREVIEW_NOTES.md.
# Preview builds are indicated with a tag that ends in -preview
# In order to effect this, you can do the following:
# scripts/release-notes.sh > PREVIEW_NOTES.md
# scripts/tag-preview-build
# If you want to replace a build, you can use -f and then
# push the update to the tag like push -f origin <tag>,
# although this should never be done!
on:
push:
@ -13,17 +16,17 @@ on:
jobs:
preview-build:
runs-on: ubuntu-20.04
outputs:
preview_version: ${{ steps.versions.outputs.PREVIEW_VERSION }}
docker_tags: ${{ steps.versions.outputs.DOCKER_TAGS }}
steps:
- name: checkout repo
uses: actions/checkout@v3
with:
node-version: '16'
- name: setup java
uses: actions/setup-java@v3
with:
node-version: '16'
distribution: 'temurin'
java-version: '17'
java-package: jdk
@ -44,40 +47,38 @@ jobs:
df -h
- name: Cache Maven packages
uses: actions/cache@v2
uses: actions/cache@v3.2.3
with:
node-version: '16'
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
- name: read versions
id: versions
run: |
set -x
PREVIEW_VERSION=$(scripts/get-preview-version.sh)
echo "PREVIEW_VERSION=${PREVIEW_VERSION}" >> $GITHUB_ENV
echo "PREVIEW_VERSION=${PREVIEW_VERSION}" >> $GITHUB_OUTPUT
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${PREVIEW_VERSION},nosqlbench/nosqlbench:preview" >> $GITHUB_ENV
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${PREVIEW_VERSION},nosqlbench/nosqlbench:preview" >> $GITHUB_OUTPUT
- name: build preview revision
run: |
mvn clean package -Drevision="${{ env.PREVIEW_VERSION }}"
- name: Setup docker buildx
uses: docker/setup-buildx-action@v1
with:
node-version: '16'
uses: docker/setup-buildx-action@v2.2.1
- name: docker hub login
uses: docker/login-action@v1
uses: docker/login-action@v2.1.0
with:
node-version: '16'
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: docker test build
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3.3.0
with:
node-version: '16'
context: .
file: Dockerfile
pull: true
@ -99,7 +100,6 @@ jobs:
- name: upload artifacts
uses: actions/upload-artifact@v1
with:
node-version: '16'
name: binaries
path: staging
@ -117,18 +117,12 @@ jobs:
id: prepare_summary
run: |
#summary=$(scripts/release-notes.sh)
summary=$(cat RELEASENOTES.md)
summary=$(cat PREVIEW_NOTES.md)
summary="${summary//'%'/'%25'}"
summary="${summary//$'\n'/'%0A'}"
summary="${summary//$'\r'/'%0D'}"
echo "release_summary=$summary" >> $GITHUB_STATE
- name: tag preview version
run: |
git tag nb5-${{ env.PREVIEW_VERSION }}
git tag -f nb5-preview
git push --tags
- name: bump minor version
run: |
scripts/bump-minor-version
@ -136,7 +130,6 @@ jobs:
- name: docker push to hub
uses: docker/build-push-action@v2
with:
node-version: '16'
context: .
platforms: linux/amd64,linux/arm64
file: Dockerfile
@ -144,125 +137,107 @@ jobs:
push: true
tags: ${{ env.DOCKER_TAGS }}
# https://github.com/softprops/action-gh-release
- name: create github release
id: create_github_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: softprops/action-gh-release@v0.1.15
if: startsWith(github.ref, 'refs/tags/')
with:
node-version: '16'
tag_name: ${{ env.PREVIEW_VERSION }}
release_name: Release ${{ env.PREVIEW_VERSION }}
# body: ${{ steps.prepare_summary.outputs.release_summary }}
# body_path: PREVIEW_NOTES.md
draft: false
prerelease: true
body: ${{ steps.prepare_summary.outputs.release_summary }}
generate_release_notes: true
name: Preview ${{ env.PREVIEW_VERSION }}
fail_on_unmatched_files: true
tag_name: ${{ env.PREVIEW_VERSION }}
token: ${{ secrets.GITHUB_TOKEN }}
files: |
nb5/target/nb5
nb5/target/nb5.jar
- name: upload nb.jar to github release
id: upload-nb-jar
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
node-version: '16'
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
asset_path: nb5/target/nb5.jar
asset_name: nb5.jar
asset_content_type: application/octet-stream
- name: upload nb binary to github release
id: upload-nb-binary
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
node-version: '16'
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
asset_path: nb5/target/nb5
asset_name: nb5
asset_content_type: application/octet-stream
# - name: create github release
# id: create_github_release
# uses: actions/create-release@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# tag_name: ${{ env.PREVIEW_VERSION }}
# release_name: Release ${{ env.PREVIEW_VERSION }}
# draft: false
# prerelease: true
# body: ${{ steps.prepare_summary.outputs.release_summary }}
#
# - name: upload nb.jar to github release
# id: upload-nb-jar
# uses: actions/upload-release-asset@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
# asset_path: nb5/target/nb5.jar
# asset_name: nb5.jar
# asset_content_type: application/octet-stream
#
# - name: upload nb binary to github release
# id: upload-nb-binary
# uses: actions/upload-release-asset@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
# asset_path: nb5/target/nb5
# asset_name: nb5
# asset_content_type: application/octet-stream
- name: Archive Test Results
if: always()
uses: actions/upload-artifact@v3
with:
node-version: '16'
name: test-results
path: |
[a-zA-Z]**/logs/*
# javadocs:
# needs: release
# runs-on: ubuntu-20.04
# steps:
# - name: set git username
# run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
# - name: set git email
# run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
# - name: download javadocs
# uses: actions/download-artifact@v3
# with:
# name: javadoc
# - run: ls -la
# - name: unpackage javadoc
# run: unzip nosqlbench-*-javadoc.jar
# - run: ls -la
# - name: Push javadocs
# uses: ad-m/github-push-action@master
# with:
# repository: nosqlbench/nosqlbench-javadoc
# github_token: ${{ secrets.GITHUB_TOKEN }}
# branch: main
#
#
# docs:
# needs: release
# runs-on: ubuntu-20.04
# steps:
#
# - name: set git username
# run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
#
# - name: set git email
# run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
#
# - name: download guidebook
# uses: actions/download-artifact@v1
# with:
# name: guidebook
# path: guidebook
#
# - run: ls -la
#
# - name: clone nosqlbench-docs
# env:
# NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
# NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
# run: |
# git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-docs.git nosqlbench-docs
# cd nosqlbench-docs
# echo "files listing"
# find .
# git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-docs.git
# git remote -v
#
# Disabling this because it will be replaced soon.
# - name: push changes
# env:
# NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
# NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
# run: |
# rsync -av --delete guidebook/guidebook/ nosqlbench-docs/docs/
# echo "docs.nosqlbench.io" > nosqlbench-docs/docs/CNAME
# cd nosqlbench-docs
# git add docs
# git add -u
# CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
# echo "found $CHANGES to push for doc updates"
# if (( $CHANGES > 0 ))
# then
# git commit -m"docs update for $GITHUB_REF"
# git push
# fi
# This triggers a preview build by cascading the tag to the builddocs repo.
# The builddocs repo then pushes to preview or release depending on the tag.
preview-docs:
needs: preview-build
runs-on: ubuntu-20.04
steps:
- name: import env vars
run: |
echo "PREVIEW_VERSION=${{ needs.preview-build.outputs.preview_version }}" >> $GITHUB_ENV
echo "DOCKER_TAGS=${{ needs.preview-build.outputs.docker_tags }}" >> $GITHUB_ENV
- name: clone nosqlbench-build-docs
run: |
git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs
cd nosqlbench-build-docs
echo "files listing"
find .
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git
git remote -v
- name: set CNAME
run: |
echo "builddocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME
- name: commit changes
run: |
cd nosqlbench-build-docs
git add exported_docs.zip
- name: tag-preview-build
run: |
cd nosqlbench-build-docs
git tag -f ${{ env.PREVIEW_VERSION }}
- name: push changes
env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
run: |
set -x
cd nosqlbench-build-docs
CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
git push -f --tags
echo "push completed"

152
.github/workflows/release.yml vendored
View File

@ -1,9 +1,12 @@
name: release
# This workflow should run for any push which is a preview build.
# Preview builds are indicated with changes to PREVIEW_NOTES.md.
# This workflow should run for any push which is a release build.
# Release builds are indicated with a tag that ends in -release
# In order to effect this, you can do the following:
# scripts/release-notes.sh > RELEASE_NOTES.md
# scripts/tag-release-build
# If you want to replace a build, you can use -f and then
# push the update to the tag like push -f origin <tag>,
# although this should never be done!
on:
push:
@ -17,13 +20,10 @@ jobs:
- name: checkout repo
uses: actions/checkout@v3
with:
node-version: '16'
- name: setup java
uses: actions/setup-java@v3
with:
node-version: '16'
distribution: 'temurin'
java-version: '17'
java-package: jdk
@ -44,9 +44,8 @@ jobs:
df -h
- name: Cache Maven packages
uses: actions/cache@v2
uses: actions/cache@v3.2.3
with:
node-version: '16'
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
@ -54,30 +53,29 @@ jobs:
- name: read versions
run: |
set -x
RELEASE_REVISION=$(scripts/get-release-version.sh)
echo "RELEASE_REVISION=${RELEASE_REVISION}" >> $GITHUB_ENV
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${RELEASE_REVISION},nosqlbench/nosqlbench:latest" >> $GITHUB_ENV
RELEASE_VERSION=$(scripts/get-release-version.sh)
RELEASE_VERSION_NUMERIC=$(scripts/get-release-version.sh -n)
RELEASE_VERSION_SHORT=$(scripts/get-release-version.sh -s)
echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV
echo "RELEASE_VERSION_NUMERIC=${RELEASE_VERSION_NUMERIC}" >> $GITHUB_ENV
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${RELEASE_VERSION_NUMERIC},nosqlbench/nosqlbench:${RELEASE_VERSION_SHORT}-latest" >> $GITHUB_ENV
- name: build release revision
run: |
mvn clean package -Drevision="${{ env.RELEASE_REVISION }}"
mvn clean package -Drevision="${{ env.RELEASE_VERSION }}"
- name: Setup docker buildx
uses: docker/setup-buildx-action@v1
with:
node-version: '16'
uses: docker/setup-buildx-action@v2.2.1
- name: docker hub login
uses: docker/login-action@v1
uses: docker/login-action@v2.1.0
with:
node-version: '16'
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: docker test build
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3.3.0
with:
node-version: '16'
context: .
file: Dockerfile
pull: true
@ -87,7 +85,7 @@ jobs:
- name: sanity check docker image
run: |
docker run --rm nosqlbench/nosqlbench:${{ env.RELEASE_REVISION }} --version
docker run --rm nosqlbench/nosqlbench:${{ env.RELEASE_VERSION_NUMERIC }} --version
- name: bundle artifacts
run: |
@ -99,40 +97,26 @@ jobs:
- name: upload artifacts
uses: actions/upload-artifact@v1
with:
node-version: '16'
name: binaries
path: staging
# - name: generate javadoc
# run: mvn javadoc:aggregate-jar
# continue-on-error: false
#
# - name: upload javadoc
# uses: actions/upload-artifact@v3
# with:
# name: javadoc
# path: target/nosqlbench-*-javadoc.jar
# - name: generate javadoc
# run: mvn javadoc:aggregate-jar
# continue-on-error: false
#
# - name: upload javadoc
# uses: actions/upload-artifact@v3
# with:
# name: javadoc
# path: target/nosqlbench-*-javadoc.jar
- name: prepare release summary
id: prepare_summary
- name: bump minor version
run: |
#summary=$(scripts/release-notes.sh)
summary=$(cat RELEASENOTES.md)
summary="${summary//'%'/'%25'}"
summary="${summary//$'\n'/'%0A'}"
summary="${summary//$'\r'/'%0D'}"
echo "release_summary=$summary" >> $GITHUB_STATE
- name: tag release
run: |
git tag nb5-${{ env.RELEASE_REVISION }}
git tag -f nb5-release
git push --tags
scripts/bump-minor-version
- name: docker push to hub
uses: docker/build-push-action@v2
with:
node-version: '16'
context: .
platforms: linux/amd64,linux/arm64
file: Dockerfile
@ -140,48 +124,62 @@ jobs:
push: true
tags: ${{ env.DOCKER_TAGS }}
# https://github.com/softprops/action-gh-release
- name: create github release
id: create_github_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: softprops/action-gh-release@v0.1.15
if: startsWith(github.ref, 'refs/tags/')
with:
node-version: '16'
tag_name: ${{ env.RELEASE_REVISION }}
release_name: Release ${{ env.RELEASE_REVISION }}
# body: ${{ steps.prepare_summary.outputs.release_summary }}
# body_path: PREVIEW_NOTES.md
draft: false
prerelease: true
body: ${{ steps.prepare_summary.outputs.release_summary }}
generate_release_notes: true
name: Release ${{ env.RELEASE_VERSION }}
fail_on_unmatched_files: true
tag_name: ${{ env.RELEASE_VERSION }}
token: ${{ secrets.GITHUB_TOKEN }}
files: |
nb5/target/nb5
nb5/target/nb5.jar
- name: upload nb.jar to github release
id: upload-nb-jar
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
node-version: '16'
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
asset_path: nb5/target/nb5.jar
asset_name: nb5.jar
asset_content_type: application/octet-stream
- name: upload nb binary to github release
id: upload-nb-binary
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
node-version: '16'
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
asset_path: nb5/target/nb5
asset_name: nb5
asset_content_type: application/octet-stream
# - name: create github release
# id: create_github_release
# uses: actions/create-release@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# tag_name: ${{ env.RELEASE_VERSION }}
# release_name: Release ${{ env.RELEASE_VERSION }}
# draft: false
# prerelease: true
# body: ${{ steps.prepare_summary.outputs.release_summary }}
#
# - name: upload nb.jar to github release
# id: upload-nb-jar
# uses: actions/upload-release-asset@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
# asset_path: nb5/target/nb5.jar
# asset_name: nb5.jar
# asset_content_type: application/octet-stream
#
# - name: upload nb binary to github release
# id: upload-nb-binary
# uses: actions/upload-release-asset@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
# asset_path: nb5/target/nb5
# asset_name: nb5
# asset_content_type: application/octet-stream
- name: Archive Test Results
if: always()
uses: actions/upload-artifact@v3
with:
node-version: '16'
name: test-results
path: |
[a-zA-Z]**/logs/*

32
PREVIEW_NOTES.md
View File

@ -1,7 +1,33 @@
- e4e94f9e8 (HEAD -> main) support version bumping
- 2b85c1cce (HEAD -> main, tag: 5.17.1-preview) versions and release action update
- 2d3263cfb make release tag distinct
- 639dbf2d6 update action version
- 1b2a773ce update files paths
- 1986b8cdb replace release action
- 7047f4486 (origin/main) remove second tagging
- 910e871a8 remove extra node version parameters
- 6b0633d4a (tag: nb5-preview, tag: nb5-5.17.1-preview) update notes filenames
- be65f812e allow forcing preview tag update
- 2a43dd077 Merge branch 'main' of github.com:nosqlbench/nosqlbench
- 6a2132669 actions testing
- 4bd0513f9 Merge pull request #950 from nosqlbench/nosqlbench-949-sigint
- 086451794 preview build testing
- 32486b3d6 unbreak the break signal
- 6d07acb8c (origin/nosqlbench-949-sigint, nosqlbench-949-sigint) unbreak the break signal
- c6aa66f50 Update DOWNLOADS.md
- 5b90831a9 Update DOWNLOADS.md
- 3e3427fca update job names
- 3f0417c96 fix preview trigger
- 29ed32d91 update revision
- 24657248f use tags only for preview and release triggers
- e254eafa8 check for main branch on preview and release
- 0ae3bd085 make preview and release tag based
- 027fc4a89 version, tagging updates
- 402154e4c docs and scripts
- ee2bda24e preview testing
- e4e94f9e8 support version bumping
- 8a2c696eb allow bumping versions
- c19b2388e (origin/prerelease-testing, origin/main, prerelease-testing) standardize on main/preview/release naming
- 5facaddfe (tag: 5.17.1-prerelease) actions testing
- c19b2388e (origin/prerelease-testing, prerelease-testing) standardize on main/preview/release naming
- 5facaddfe actions testing
- 33c55809d actions testing
- fcdddf307 simplify version
- ac7f5762c actions cleanup

44
adapter-cqld4/pom.xml
View File

@ -50,7 +50,7 @@
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>3.6.1</version>
<version>3.6.2</version>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@ -140,48 +140,6 @@
<followSymLinks>false</followSymLinks>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>

27
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,13 +16,13 @@
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import java.io.File;
import java.io.InputStream;
@ -36,14 +36,17 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml";
public CGDefaultCqlBindings() {
String yamlContent = NBIO.all()
Content<?> content = NBIO.all()
.name(DEFAULT_BINDINGS_FILE)
.first()
.map(Content::asString)
.or(() -> loadLocal(DEFAULT_BINDINGS_FILE))
.orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE));
StmtsDocList stmtsDocs = StatementsLoader.loadString(yamlContent, Map.of());
this.bindings = stmtsDocs.getDocBindings();
.or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first())
.orElseThrow(
() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE +
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)
);
OpsDocList opsDocList = OpsLoader.loadContent(content, Map.of());
this.bindings = opsDocList.getDocBindings();
}
private Optional<String> loadLocal(String path) {

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CqlSchemaStatsParser.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,7 +19,7 @@ package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.core.CGTableStats;
import org.apache.commons.math4.util.Pair;
import org.apache.commons.math4.legacy.core.Pair;
import java.io.BufferedReader;
import java.io.IOException;

14
adapter-cqld4/src/main/resources/activities/baselines/cql-iot.yaml
View File

@ -6,13 +6,13 @@ description: |
scenarios:
default:
schema: run driver=cql tags==phase:schema threads==1 cycles==UNDEF
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
astra:
schema: run driver=cql tags==phase:schema-astra threads==1 cycles==UNDEF
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
schema: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
params:
instrument: TEMPLATE(instrument,false)
@ -20,7 +20,7 @@ params:
bindings:
machine_id: Mod(<<sources:10000>>); ToHashedUUID() -> java.util.UUID
sensor_name: HashedLineToString('data/variable_words.txt')
time: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); ToDate()
time: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); ToJavaInstant();
cell_timestamp: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); Mul(1000L)
sensor_value: Normal(0.0,5.0); Add(100.0) -> double
station_id: Div(<<sources:10000>>);Mod(<<stations:100>>); ToHashedUUID() -> java.util.UUID

138
adapter-cqld4/src/main/resources/activities/baselines/cql-keyvalue.yaml
View File

@ -1,14 +1,22 @@
description: A workload with only text keys and text values
description: |
A workload with only text keys and text values.
This workload is rather simple but it is useful for demonstration purposes.
If you want a more thorough example, you might be interested in the
baselinesv2/cql-* workloads.
defaults:
TEMPLATE(keyspace,baselines)
TEMPLATE(table,keyvalue)
scenarios:
default:
schema: run driver=cql tags==phase:schema threads==1 cycles==UNDEF
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags=='block:main-.*' cycles===TEMPLATE(main-cycles,10000000) threads=auto
astra:
schema: run driver=cql tags==phase:schema-astra threads==1 cycles==UNDEF
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
schema: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags=='block:main-.*' cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:
seq_key: Mod(<<keycount:1000000000>>); ToString() -> String
@ -17,86 +25,60 @@ bindings:
rw_value: Hash(); <<valdist:Uniform(0,1000000000)->int>>; ToString() -> String
blocks:
- name: schema
tags:
phase: schema
params:
prepared: false
statements:
- create-keyspace: |
create keyspace if not exists <<keyspace:baselines>>
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '<<rf:1>>'}
AND durable_writes = true;
tags:
name: create-keyspace
- create-table: |
create table if not exists <<keyspace:baselines>>.<<table:keyvalue>> (
schema:
ops:
create-keyspace:
raw: |
create keyspace if not exists TEMPLATE(keyspace,baselines)
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 'TEMPLATE(rf,1)>>'}
AND durable_writes = true;
create-table:
raw: |
create table if not exists TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) (
key text,
value text,
PRIMARY KEY (key)
);
schema-astra:
ops:
create-table:
raw: |
create table if not exists TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) (
key text,
value text,
PRIMARY KEY (key)
);
tags:
name: create-table
- name: schema-astra
tags:
phase: schema-astra
rampup:
params:
prepared: false
statements:
- create-table: |
create table if not exists <<keyspace:baselines>>.<<table:keyvalue>> (
key text,
value text,
PRIMARY KEY (key)
);
tags:
name: create-table-astra
- name: rampup
tags:
phase: rampup
params:
cl: <<write_cl:LOCAL_QUORUM>>
statements:
- rampup-insert: |
insert into <<keyspace:baselines>>.<<table:keyvalue>>
cl: TEMPLATE(write_cl,LOCAL_QUORUM)
ops:
rampup-insert:
prepared: |
insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
(key, value)
values ({seq_key},{seq_value});
tags:
name: rampup-insert
- name: verify
tags:
phase: verify
type: read
params:
cl: <<read_cl:LOCAL_QUORUM>>
statements:
- verify-select: |
select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={seq_key};
verify-fields: key->seq_key, value->seq_value
tags:
name: verify
- name: main-read
tags:
phase: main
type: read
# verify:
# params:
# cl: <<read_cl:LOCAL_QUORUM>>
# ops:
# verify-select:
# prepared: |
# select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={seq_key};
# verify-fields: key->seq_key, value->seq_value
main-read:
params:
ratio: 5
cl: <<read_cl:LOCAL_QUORUM>>
statements:
- main-select: |
select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={rw_key};
tags:
name: main-select
- name: main-write
tags:
phase: main
type: write
cl: TEMPLATE(read_cl,LOCAL_QUORUM)
ops:
main-select:
prepared: |
select * from TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) where key={rw_key};
main-write:
params:
ratio: 5
cl: <<write_cl:LOCAL_QUORUM>>
statements:
- main-insert: |
insert into <<keyspace:baselines>>.<<table:keyvalue>>
cl: TEMPLATE(write_cl,LOCAL_QUORUM)
ops:
main-insert:
prepared: |
insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
(key, value) values ({rw_key}, {rw_value});
tags:
name: main-insert

11
adapter-diag/src/main/java/io/nosqlbench/adapter/diag/DiagDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,9 +17,10 @@
package io.nosqlbench.adapter.diag;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -101,7 +102,7 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> params) {
return StatementsLoader.loadString("ops: 'log:level=INFO'", params).getStmts();
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) {
return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getOps();
}
}

49
adapter-dynamodb/pom.xml
View File

@ -48,53 +48,4 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

47
adapter-http/pom.xml
View File

@ -33,53 +33,6 @@
This provides the ability to generate HTTP requests with nosqlbench
</description>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>

17
adapter-http/src/test/java/io/nosqlbench/adapter/http/HttpOpMapperTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,9 +19,10 @@ package io.nosqlbench.adapter.http;
import io.nosqlbench.adapter.http.core.HttpOpMapper;
import io.nosqlbench.adapter.http.core.HttpSpace;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
import io.nosqlbench.engine.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
@ -51,9 +52,9 @@ public class HttpOpMapperTest {
}
private static ParsedOp parsedOpFor(String yaml) {
StmtsDocList docs = StatementsLoader.loadString(yaml, Map.of());
OpTemplate stmtDef = docs.getStmts().get(0);
ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor()));
OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
OpTemplate opTemplate = docs.getOps().get(0);
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(adapter.getPreprocessor()));
return parsedOp;
}
@ -101,7 +102,7 @@ public class HttpOpMapperTest {
@Test
public void testRFCFormBody() {
ParsedOp pop = parsedOpFor("""
statements:
ops:
- s1: |
get http://localhost/
@ -117,7 +118,7 @@ public class HttpOpMapperTest {
// This can not be fully resolved in the unit testing context, but it could be
// in the integrated testing context. It is sufficient to verify parsing here.
ParsedOp pop = parsedOpFor("""
statements:
ops:
- s1: |
{method} {scheme}://{host}/{path}?{query} {version}
Header1: {header1val}

48
adapter-kafka/pom.xml
View File

@ -34,55 +34,9 @@
</description>
<properties>
<kafka.version>3.3.1</kafka.version>
<kafka.version>3.3.2</kafka.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>

46
adapter-mongodb/pom.xml
View File

@ -46,50 +46,4 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

47
adapter-pulsar/pom.xml
View File

@ -91,51 +91,4 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

44
adapter-s4j/pom.xml
View File

@ -100,50 +100,6 @@
<version>1.7.1</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

47
adapter-stdout/pom.xml
View File

@ -53,50 +53,9 @@
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

8
adapter-stdout/src/main/java/io/nosqlbench/adapter/stdout/StdoutDriverAdapter.java
View File

@ -18,7 +18,7 @@ package io.nosqlbench.adapter.stdout;
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -59,8 +59,8 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
Set<String> activeBindingNames = new LinkedHashSet<>(stmtsDocList.getDocBindings().keySet());
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
Set<String> activeBindingNames = new LinkedHashSet<>(opsDocList.getDocBindings().keySet());
if (activeBindingNames.size()==0) {
logger.warn("Unable to synthesize op for driver=" + this.getAdapterName() + " with zero bindings.");
@ -88,7 +88,7 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
return List.of();
}
OpData op = new OpData("synthetic", "synthetic", Map.of(), stmtsDocList.getDocBindings(), cfg,
OpData op = new OpData("synthetic", "synthetic", Map.of(), opsDocList.getDocBindings(), cfg,
Map.of("stmt", genStatementTemplate(filteredBindingNames, cfg)));
return List.of(op);

34
virtdata-lib-curves4/src/main/java/org/apache/commons/package-info.java → adapter-stdout/src/test/java/io/nosqlbench/adapter/stdout/TemplateFormatTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,12 +14,26 @@
* limitations under the License.
*/
/**
* This package is an import of some org.apache.commons source trees.
* Once the state of the newer o.a.c.math4 and distributions modules is stable,
* these will be removed and the release artifacts will be used.
* The enclosed packages were included here as the least worst of several options,
* given the confluence of JPMS, the status of the math4 repos, and the need
* to access stable CDF functions across the extant distributions.
*/
package org.apache.commons;
package io.nosqlbench.adapter.stdout;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class TemplateFormatTest {
@Test
public void testBasicFormats() {
assertThat(TemplateFormat.json.format(true, List.of("f1","f2")))
.isEqualTo("""
{
"f1":"{f1}",
"f2":"{f2}"
}
""");
}
}

47
adapter-tcp/pom.xml
View File

@ -51,51 +51,4 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

8
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpclient/TcpClientDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,7 +21,7 @@ import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -58,8 +58,8 @@ public class TcpClientDriverAdapter extends BaseDriverAdapter<TcpClientOp, TcpCl
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(stmtsDocList, cfg);
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(opsDocList, cfg);
}
}

8
adapter-tcp/src/main/java/io/nosqlbench/adapter/tcpserver/TcpServerDriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,7 +20,7 @@ import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
@ -57,7 +57,7 @@ public class TcpServerDriverAdapter extends BaseDriverAdapter<TcpServerOp, TcpSe
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(stmtsDocList, cfg);
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> cfg) {
return adap.getSyntheticOpTemplates(opsDocList, cfg);
}
}

55
adapters-api/pom.xml
View File

@ -54,54 +54,21 @@
<version>${revision}</version>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>sjsonnet_2.13</artifactId>
<version>0.4.3</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.13.10</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

150
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig;
import com.amazonaws.util.StringInputStream;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import scala.Option;
import sjsonnet.DefaultParseCache;
import sjsonnet.SjsonnetMain;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class OpsLoader {
private final static Logger logger = LogManager.getLogger(OpsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"};
public static OpsDocList loadContent(Content<?> content, Map<String, String> params) {
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(content.getURI());
return loadString(content.get().toString(), fmt, params, content.getURI());
}
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS;
Content<?> foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());
}
public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
logger.trace(() -> "Applying string transformer to data:" + sourceData);
StrInterpolator transformer = new StrInterpolator(params);
String data = transformer.apply(sourceData);
if (srcuri!=null) {
logger.info("workload URI: '" + srcuri + "'");
}
RawOpsLoader loader = new RawOpsLoader(transformer);
RawOpsDocList rawOpsDocList = switch (fmt) {
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
case yaml, json -> loader.loadString(data);
case inline, stmt -> RawOpsDocList.forSingleStatement(data);
};
// TODO: itemize inline to support ParamParser
OpsDocList layered = new OpsDocList(rawOpsDocList);
transformer.checkpointAccesses().forEach((k, v) -> {
layered.addTemplateVariable(k, v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
private static String evaluateJsonnet(URI uri, Map<String, ?> params) {
List<String> injected = new LinkedList<>(List.of(Path.of(uri).toString()));
params.forEach((k,v) -> {
if (v instanceof CharSequence cs) {
injected.addAll(List.of("--ext-str",k+"="+cs));
}
});
var stdoutBuffer = new ByteArrayOutputStream();
var stderrBuffer = new ByteArrayOutputStream();
var stdoutStream = new PrintStream(stdoutBuffer);
var stderrStream = new PrintStream(stderrBuffer);
StringInputStream inputStream;
try {
inputStream = new StringInputStream("");
} catch (Exception e) {
throw new RuntimeException("Error building input stream for jsonnet:" + e, e);
}
int resultStatus = SjsonnetMain.main0(
injected.toArray(new String[0]),
new DefaultParseCache(),
inputStream,
stdoutStream,
stderrStream,
new os.Path(Path.of(System.getProperty("user.dir"))),
Option.empty(),
Option.empty()
);
String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8);
String stderrOutput = stderrBuffer.toString(StandardCharsets.UTF_8);
if ("jsonnet".equals(String.valueOf(params.get("dryrun")))) {
logger.info("dryrun=jsonnet, dumping result to stdout and stderr:");
System.out.println(stdoutOutput);
System.err.println(stderrOutput);
if (resultStatus==0 && stderrOutput.isEmpty()) {
logger.info("no errors detected during jsonnet evaluation.");
System.exit(0);
} else {
logger.error("ERRORS detected during jsonnet evaluation:\n" + stderrOutput);
System.exit(2);
}
}
if (!stderrOutput.isEmpty()) {
BasicError error = new BasicError("stderr output from jsonnet preprocessing: " + stderrOutput);
if (resultStatus!=0) {
throw error;
} else {
logger.warn(error.toString(),error);
}
}
logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines.");
logger.trace("jsonnet result:\n" + stdoutOutput);
return stdoutOutput;
}
}

8
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedStmtOp.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Function;
public class ParsedStmtOp {
public class ParsedOpTemplate {
private final OpTemplate optpl;
private final ParsedStringTemplate parsed;
@ -37,13 +37,13 @@ public class ParsedStmtOp {
*
* @param optpl An existing statement def as read from the YAML API.
*/
public ParsedStmtOp(OpTemplate optpl) {
public ParsedOpTemplate(OpTemplate optpl) {
this.optpl = optpl;
String transformed = getStmt();
parsed = new ParsedStringTemplate(transformed, optpl.getBindings());
}
public ParsedStmtOp orError() {
public ParsedOpTemplate orError() {
if (hasError()) {
throw new RuntimeException("Unable to parse statement: " + this);
}

99
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -1,99 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.Optional;
public class StatementsLoader {
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
StrInterpolator transformer = new StrInterpolator(params);
RawStmtsLoader loader = new RawStmtsLoader(transformer);
RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
StmtsDocList layered = new StmtsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
public static StmtsDocList loadStmt(
Logger logger,
String statement,
Map<String,?> params
) {
StrInterpolator transformer = new StrInterpolator(params);
statement = transformer.apply(statement);
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
public static StmtsDocList loadContent(
Logger logger,
Content<?> content,
Map<String,String> params
) {
return loadString(content.get().toString(),params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
Map<String,?> params,
String... searchPaths) {
RawStmtsDocList list = null;
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(content,params);
}
public static StmtsDocList loadPath(
Logger logger,
String path,
String... searchPaths) {
return loadPath(logger, path, Map.of(), searchPaths);
}
}

36
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StatementsOwner.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpsOwner.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,21 +20,21 @@ import io.nosqlbench.api.errors.BasicError;
import java.util.*;
public class StatementsOwner extends RawStmtFields {
public class OpsOwner extends RawOpFields {
private final static List<String> stmtsFieldNames = List.of("op","ops","operation","statement","statements");
private final static List<String> opsFieldNames = List.of("op","ops","operation","statement","statements");
private List<RawStmtDef> rawStmtDefs = new ArrayList<>();
private List<RawOpDef> rawOpDefs = new ArrayList<>();
public StatementsOwner() {
public OpsOwner() {
}
public List<RawStmtDef> getRawStmtDefs() {
return rawStmtDefs;
public List<RawOpDef> getRawOpDefs() {
return rawOpDefs;
}
public void setRawStmtDefs(List<RawStmtDef> rawStmtDefs) {
this.rawStmtDefs = rawStmtDefs;
public void setRawStmtDefs(List<RawOpDef> rawOpDefs) {
this.rawOpDefs = rawOpDefs;
}
public void setFieldsByReflection(Map<String, Object> propsmap) {
@ -43,7 +43,7 @@ public class StatementsOwner extends RawStmtFields {
super.setFieldsByReflection(propsmap);
HashSet<String> found = new HashSet<>();
for (String fname : stmtsFieldNames) {
for (String fname : opsFieldNames) {
if (propsmap.containsKey(fname)) {
found.add(fname);
}
@ -52,25 +52,25 @@ public class StatementsOwner extends RawStmtFields {
throw new BasicError("You used " + found + " as an op name, but only one of these is allowed.");
}
if (found.size()==1) {
Object stmtsFieldValue = propsmap.remove(found.iterator().next());
setStatementsFieldByType(stmtsFieldValue);
Object opsFieldValue = propsmap.remove(found.iterator().next());
setOpsFieldByType(opsFieldValue);
}
super.setFieldsByReflection(propsmap);
}
@SuppressWarnings("unchecked")
public void setStatementsFieldByType(Object object) {
public void setOpsFieldByType(Object object) {
if (object instanceof List) {
List<Object> stmtList = (List<Object>) object;
List<RawStmtDef> defs = new ArrayList<>(stmtList.size());
List<RawOpDef> defs = new ArrayList<>(stmtList.size());
for (int i = 0; i < stmtList.size(); i++) {
String defaultName = "stmt" + (i + 1);
Object o = stmtList.get(i);
if (o instanceof String) {
defs.add(new RawStmtDef(defaultName, (String) o));
defs.add(new RawOpDef(defaultName, (String) o));
} else if (o instanceof Map) {
RawStmtDef def = new RawStmtDef(defaultName, (Map<String, Object>) o);
RawOpDef def = new RawOpDef(defaultName, (Map<String, Object>) o);
defs.add(def);
} else {
throw new RuntimeException("Can not construct stmt def from object type:" + o.getClass());
@ -105,9 +105,9 @@ public class StatementsOwner extends RawStmtFields {
entries.getValue().getClass() + "', only maps and strings are recognized.");
}
}
setStatementsFieldByType(itemizedMaps);
setOpsFieldByType(itemizedMaps);
} else if (object instanceof String) {
setStatementsFieldByType(Map.of("stmt1", (String) object));
setOpsFieldByType(Map.of("stmt1", (String) object));
} else {
throw new RuntimeException("Unknown object type: " + object.getClass());
}

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtDef.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpDef.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,22 +24,22 @@ import java.util.*;
/**
* See specification for what this should do in UniformWorkloadSpecificationTest
*/
public class RawStmtDef extends RawStmtFields {
public class RawOpDef extends RawOpFields {
private Object op;
private final static List<String> opFieldSynonyms = List.of("stmt", "statement", "op", "operation");
public RawStmtDef() {
public RawOpDef() {
}
public RawStmtDef(String name, String op) {
public RawOpDef(String name, String op) {
setName(name);
this.op = op;
}
@SuppressWarnings("unchecked")
public RawStmtDef(String defaultName, Map<String, Object> map) {
public RawOpDef(String defaultName, Map<String, Object> map) {
setFieldsByReflection(map);
if (this.getName() == null || this.getName().isEmpty()) {
this.setName(defaultName);

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtFields.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpFields.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -25,15 +25,15 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
public class RawStmtFields extends Tags {
private final static Logger logger = LogManager.getLogger(RawStmtFields.class);
public class RawOpFields extends Tags {
private final static Logger logger = LogManager.getLogger(RawOpFields.class);
private String name = "";
private String desc = "";
private final Map<String, String> bindings = new LinkedHashMap<>();
private final Map<String, Object> params = new LinkedHashMap<>();
public RawStmtFields() {
public RawOpFields() {
}
public String getDesc() {
@ -80,7 +80,7 @@ public class RawStmtFields extends Tags {
this.params.putAll(config);
}
public void applyBlockParams(RawStmtFields other) {
public void applyBlockParams(RawOpFields other) {
setName(other.getName());
setBindings(other.getBindings());
setTags(other.getTags());

8
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsBlock.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsBlock.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,12 +20,12 @@ import java.util.Map;
/**
* A StmtsDef contains a list of rawStmts, as well as all of the optional
* block parameters that can be assigned to {@link RawStmtFields}, which includes
* block parameters that can be assigned to {@link RawOpFields}, which includes
* a name, config values, data bindings, and filtering tags.
*/
public class RawStmtsBlock extends StatementsOwner {
public class RawOpsBlock extends OpsOwner {
public RawStmtsBlock() {}
public RawOpsBlock() {}
public void setFieldsByReflection(Map<String, Object> propsmap) {
super.setFieldsByReflection(propsmap);

63
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDoc.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDoc.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,29 +22,20 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* A statements doc can have both a list of statement blocks and/or a
* list of statements. It can also have all the block parameters
* assignable to {@link RawStmtFields}.
* <p>
* The reason for having support both statements or statement blocks
* is merely convenience. If you do not need or want to deal with the
* full blocks format, the extra structure gets in the way.
*/
public class RawStmtsDoc extends StatementsOwner {
public class RawOpsDoc extends OpsOwner {
private RawScenarios scenarios = new RawScenarios();
private final List<RawStmtsBlock> blocks = new ArrayList<>();
private final List<RawOpsBlock> blocks = new ArrayList<>();
private String versionRegex = ".+";
// no-args ctor is required
public RawStmtsDoc() {
public RawOpsDoc() {
}
public static RawStmtsDoc forSingleStatement(String statement) {
RawStmtsDoc rawStmtsDoc = new RawStmtsDoc();
rawStmtsDoc.setStatementsFieldByType(statement);
return rawStmtsDoc;
public static RawOpsDoc forSingleStatement(String statement) {
RawOpsDoc rawOpsDoc = new RawOpsDoc();
rawOpsDoc.setOpsFieldByType(statement);
return rawOpsDoc;
}
public void setFieldsByReflection(Map<String, Object> properties) {
@ -63,9 +54,9 @@ public class RawStmtsDoc extends StatementsOwner {
for (Object blockData : blockList) {
if (blockData instanceof Map) {
Map<String, Object> blockDataMap = (Map<String, Object>) blockData;
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawStmtsBlock);
RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawOpsBlock);
} else {
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
}
@ -77,10 +68,10 @@ public class RawStmtsDoc extends StatementsOwner {
Object blockData = entry.getValue();
if (blockData instanceof Map) {
Map<String, Object> blockDataMap = (Map<String, Object>) blockData;
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setName(blockName);
rawStmtsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawStmtsBlock);
RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setName(blockName);
rawOpsBlock.setFieldsByReflection(blockDataMap);
blocks.add(rawOpsBlock);
} else {
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
}
@ -100,26 +91,24 @@ public class RawStmtsDoc extends StatementsOwner {
}
/**
* Return the list of statement blocks in this RawStmtsDoc.
* If raw statements are defined on this RawStmtsDoc, then a single
* StmtBlock containing those statements is prepended to the block list.
* Otherwise, the list of StmtBlocks is returned as-is.
* If raw ops are defined on this RawOpsDoc, then a single RawOpsBlock containing those op templates is prepended to
* the block list. Otherwise, the list of RawOpsBlocks is returned as-is.
*
* @return all logical statement blocks containing statements
* @return all logical ops blocks
*/
public List<RawStmtsBlock> getBlocks() {
List<RawStmtsBlock> stmtBlocks = new ArrayList<>();
if (!getRawStmtDefs().isEmpty()) {
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
rawStmtsBlock.setName("block0");
rawStmtsBlock.setRawStmtDefs(getRawStmtDefs());
stmtBlocks.add(rawStmtsBlock);
public List<RawOpsBlock> getBlocks() {
List<RawOpsBlock> stmtBlocks = new ArrayList<>();
if (!getRawOpDefs().isEmpty()) {
RawOpsBlock rawOpsBlock = new RawOpsBlock();
rawOpsBlock.setName("block0");
rawOpsBlock.setRawStmtDefs(getRawOpDefs());
stmtBlocks.add(rawOpsBlock);
}
stmtBlocks.addAll(this.blocks);
return stmtBlocks;
}
public void setBlocks(List<RawStmtsBlock> blocks) {
public void setBlocks(List<RawOpsBlock> blocks) {
this.blocks.clear();
this.blocks.addAll(blocks);
}

48
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsDocList.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.List;
public class RawOpsDocList {
private final List<RawOpsDoc> rawOpsDocList;
public RawOpsDocList(List<RawOpsDoc> rawOpsDocList) {
this.rawOpsDocList = rawOpsDocList;
}
public static RawOpsDocList forSingleStatement(String statement) {
RawOpsDoc rawOpsDoc = RawOpsDoc.forSingleStatement(statement);
return new RawOpsDocList(List.of(rawOpsDoc));
}
public static RawOpsDocList none() {
return new RawOpsDocList(List.of());
}
public List<RawOpsDoc> getOpsDocs() {
return rawOpsDocList;
}
public String toString() {
int docs = rawOpsDocList.size();
int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum();
long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawOpDefs().stream()).count();
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
}
}

46
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsLoader.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -21,6 +21,7 @@ import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
@ -28,18 +29,18 @@ import org.snakeyaml.engine.v2.api.LoadSettings;
import java.util.*;
import java.util.function.Function;
public class RawStmtsLoader {
public class RawOpsLoader {
private final static Logger logger = LogManager.getLogger(RawOpsLoader.class);
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
List<Function<String, String>> stringTransformers = new ArrayList<>();
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
public RawStmtsLoader(Function<String,String> transformer) {
public RawOpsLoader(Function<String,String> transformer) {
addTransformer(transformer);
}
public RawStmtsLoader() {
public RawOpsLoader() {
addTransformer(new StrInterpolator());
}
@ -47,11 +48,10 @@ public class RawStmtsLoader {
Collections.addAll(this.transformers, newTransformer);
}
public RawStmtsDocList loadString(Logger logger, final String originalData) {
public RawOpsDocList loadString(final String originalData) {
logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
String data = originalData;
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
for (Function<String, String> transformer : transformers) {
data = transformer.apply(data);
}
@ -60,11 +60,10 @@ public class RawStmtsLoader {
throw t;
}
return parseYaml(logger, data);
return parseYaml(data);
}
public RawStmtsDocList loadPath(
Logger logger,
public RawOpsDocList loadPath(
String path,
String... searchPaths) {
@ -72,22 +71,22 @@ public class RawStmtsLoader {
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(logger, data);
return loadString(data);
} catch (Exception e) {
throw new RuntimeException("error while reading file " + path, e);
}
}
private RawStmtsDocList parseYaml(Logger logger, String data) {
public RawOpsDocList parseYaml(String data) {
LoadSettings loadSettings = LoadSettings.builder().build();
Load yaml = new Load(loadSettings);
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
List<RawOpsDoc> newDocList = new ArrayList<>();
for (Object object : objects) {
if (object instanceof Map) {
RawStmtsDoc doc = new RawStmtsDoc();
RawOpsDoc doc = new RawOpsDoc();
Map<String, Object> docfields = (Map<String, Object>) object;
doc.setFieldsByReflection(docfields);
if (docfields.size()>0) {
@ -96,24 +95,11 @@ public class RawStmtsLoader {
newDocList.add(doc);
} else {
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
}
}
RawStmtsDocList rawStmtsDocList = new RawStmtsDocList(newDocList);
return rawStmtsDocList;
}
protected String applyTransforms(Logger logger, String data) {
for (Function<String, String> xform : stringTransformers) {
try {
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform);
data = xform.apply(data);
} catch (Exception e) {
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
throw t;
}
}
return data;
RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList);
return rawOpsDocList;
}
}

48
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsDocList.java
View File

@ -1,48 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import java.util.List;
public class RawStmtsDocList {
private final List<RawStmtsDoc> rawStmtsDocList;
public RawStmtsDocList(List<RawStmtsDoc> rawStmtsDocList) {
this.rawStmtsDocList = rawStmtsDocList;
}
public static RawStmtsDocList forSingleStatement(String statement) {
RawStmtsDoc rawStmtsDoc = RawStmtsDoc.forSingleStatement(statement);
return new RawStmtsDocList(List.of(rawStmtsDoc));
}
public static RawStmtsDocList none() {
return new RawStmtsDocList(List.of());
}
public List<RawStmtsDoc> getStmtsDocs() {
return rawStmtsDocList;
}
public String toString() {
int docs = rawStmtsDocList.size();
int blocks = rawStmtsDocList.stream().map(RawStmtsDoc::getBlocks).mapToInt(List::size).sum();
long optemplates = rawStmtsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count();
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
}
}

4
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@ -76,7 +76,7 @@ public class RawYamlLoader {
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
Load yaml = new Load(LoadSettings.builder().build());
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
List<RawOpsDoc> newDocList = new ArrayList<>();
List<Map<String,Object>> maps = new ArrayList<>();
@ -84,7 +84,7 @@ public class RawYamlLoader {
if (object instanceof Map) {
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
} else {
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
}
}
return maps;

24
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDef.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef;
import io.nosqlbench.api.errors.BasicError;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -27,15 +27,15 @@ import java.util.*;
public class OpDef extends OpTemplate {
private final static Logger logger = LogManager.getLogger(OpDef.class);
private final RawStmtDef rawStmtDef;
private final StmtsBlock block;
private final RawOpDef rawOpDef;
private final OpsBlock block;
private final LinkedHashMap<String, Object> params;
private final LinkedHashMap<String, String> bindings;
private final LinkedHashMap<String, String> tags;
public OpDef(StmtsBlock block, RawStmtDef rawStmtDef) {
public OpDef(OpsBlock block, RawOpDef rawOpDef) {
this.block = block;
this.rawStmtDef = rawStmtDef;
this.rawOpDef = rawOpDef;
this.params = composeParams();
this.bindings = composeBindings();
this.tags = composeTags();
@ -43,12 +43,12 @@ public class OpDef extends OpTemplate {
@Override
public String getName() {
return block.getName() + "--" + rawStmtDef.getName();
return block.getName() + "--" + rawOpDef.getName();
}
@Override
public Optional<Map<String, Object>> getOp() {
Object op = rawStmtDef.getOp();
Object op = rawOpDef.getOp();
if (op == null) {
return Optional.empty();
}
@ -75,7 +75,7 @@ public class OpDef extends OpTemplate {
}
private LinkedHashMap<String, String> composeBindings() {
MultiMapLookup<String> lookup = new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings());
MultiMapLookup<String> lookup = new MultiMapLookup<>(rawOpDef.getBindings(), block.getBindings());
return new LinkedHashMap<>(lookup);
}
@ -85,7 +85,7 @@ public class OpDef extends OpTemplate {
}
private LinkedHashMap<String, Object> composeParams() {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpDef.getParams(), block.getParams());
LinkedHashMap<String, Object> params = new LinkedHashMap<>(lookup);
return params;
}
@ -98,7 +98,7 @@ public class OpDef extends OpTemplate {
}
private LinkedHashMap<String, String> composeTags() {
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags()));
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags()));
tagsWithName.put("name",getName());
tagsWithName.put("block",block.getName());
return tagsWithName;
@ -111,7 +111,7 @@ public class OpDef extends OpTemplate {
@Override
public String getDesc() {
return rawStmtDef.getDesc();
return rawOpDef.getDesc();
}
}

142
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -32,118 +32,17 @@ import java.util.Optional;
import java.util.function.Function;
/**
* <p>The OpTemplate is the developer's view of the operational templates that users
* provide in YAML or some other structured format.</p>
*
* <H2>Terms</H2>
* Within this documentation, the word <i>OpTemplate</i> will refer to the template API and
* semantics. The word <i>user template</i> will refer to the configuration data as provided
* by a user.
*
* <p>OpTemplates are the native Java representation of the user templates that specify how to
* make an executable operation. OpTemplates are not created for each operation, but are used
* to create an mostly-baked intermediate form commonly known as a <i>ready op</i>.
* It is the intermediate form which is used to create an instance of an executable
* op in whichever way is the most appropriate and efficient for a given driver.</p>
*
* <p>This class serves as the canonical documentation and API for how user templates
* are mapped into a fully resolved OpTemplate. User-provided op templates can be
* any basic data structure, and are often provided as part of a YAML workload file.
* The description below will focus on structural rules rather than any particular
* encoding format. The types used are fairly universal and easy to map from one
* format to another.</p>
*
*
* <p>A long-form introduction to this format is included in the main NoSQLBench docs
* at <a href="http://docs.nosqlbench.io">docs.nosqlbench.io</a>
* under the <I>Designing Workloads</I> section.</p>
*
* <p>A few structural variations are allowed -- No specific form enforced. The reasons for this are:
* 1) It is generally obvious what as user wants to do from a given layout. 2) Data structure
* markup is generally frustrating and difficult to troubleshoot. 3) The conceptual domain of
* NB op construction is well-defined enough to avoid ambiguity.</p>
*
* <H2>Type Conventions</H2>
*
* For the purposes of simple interoperability, the types used at this interface boundary should
* be limited to common scalar types -- numbers and strings, and simple structures like maps and lists.
* The basic types defined for ECMAScript should eventually be supported, but no domain-specific
* objects which would require special encoding or decoding rules should be used.
*
* <H2>Standard Properties</H2>
*
* Each op template can have these standard properties:
* <UL>
* <LI>name - every op template has a name, even if it is auto generated for you. This is used to
* name errors in the log, to name metrics in telemetry, and so on.</LI>
* <LI>description - an optional description, defaulted to "".</LI>
* <LI>statement - An optional string value which represents an opaque form of the body of
* an op template</LI>
* <LI>params - A string-object map of zero or more named parameters, where the key is taken as the parameter
* name and the value is any simple object form as limited by type conventions above.
* <LI>bindings - A map of binding definitions, where the string key is taken as the anchor name, and the
* string value is taken as the binding recipe.</LI>
* <LI>tags - A map of tags, with string names and values</LI>
* </UL>
*
* The user-provided definition of an op template should capture a blueprint of an operation to be executed by
* a native driver. As such, you need either a statement or a set of params which can describe what
* specific type should be constructed. The rules on building an executable operation are not enforced
* by this API. Yet, responsible NB driver developers will clearly document what the rules
* are for specifying each specific type of operation supported by an NB driver with examples in YAML format.
*
* <H2>OpTemplate Construction Rules</H2>
*
* <p>The available structural forms follow a basic set of rules for constructing the OpTemplate in a consistent way.
* <OL>
* <LI>A collection of user-provided op templates is provided as a string, a list or a map.</LI>
* <LI>All maps are order-preserving, like {@link java.util.LinkedHashMap}</LI>
* <LI>For maps, the keys are taken as the names of the op template instances.</LI>
* <LI>The content of each op template can be provided as a string or as a map.</LI>
* <OL>
* <LI>If the op template entry is provided as a string, then the OpTemplate is constructed as having only a single
* <i>statement</i> property (in addition to defaults within scope).
* as provided by OpTemplate API.</LI>
* <LI>If the op template entry is provided as a map, then the OpTemplate is constructed as having all of the
* named properties defined in the standard properties above.
* Any entry in the template which is not a reserved word is assigned to the params map as a parameter, in whatever structured
* type is appropriate (scalar, lists, maps).</LI>
* </LI>
* <p>
* The OpTemplate is a structurally normalized type which presents the user-provided op template to the NoSQLBench
* loading and templating mechanisms. This type is not generally used directly for new driver development. It is the
* backing data which is used by {@link io.nosqlbench.engine.api.templating.ParsedOp}, which is used in drivers to map
* op templates to function to be used for a given cycle value.
* </p>
* </OL>
*
* <H2>Example Forms</H2>
* The valid forms are shown below as examples.
*
* <H3>One String Statement</H3>
* <pre>{@code
* statement: statement
* }</pre>
*
* <H3>List of Templates</H3>
* <pre>{@code
* statements:
* - statement1
* - statement2
* }</pre>
*
* <H3>List of Maps</H3>
* <pre>{@code
* statements:
* - name: name1
* stmt: statement body
* params:
* p1: v1
* p2: v2
* }</pre>
*
* <H3>List Of Condensed Maps</H3>
* <pre>{@code
* statements:
* - name1: statement body
* p1: v1
* p2: v2
* }</pre>
* <p>
* This is part of the implementation of the NoSQLBench <em>Uniform Workload Specification</em>. Check the tests
* for UniformWorkloadSpecification directly to see how this specification is tested and documented.
* </p>
*/
public abstract class OpTemplate implements Tagged {
@ -170,6 +69,7 @@ public abstract class OpTemplate implements Tagged {
/**
* Return a map of tags for this statement. Implementations are required to
* add a tag for "name" automatically when this value is set during construction.
*
* @return A map of assigned tags for the op, with the name added as an auto-tag.
*/
public abstract Map<String, String> getTags();
@ -208,7 +108,7 @@ public abstract class OpTemplate implements Tagged {
if (defaultValue.getClass().isAssignableFrom(value.getClass())) {
return (V) value;
} else {
return NBTypeConverter.convertOr(value,defaultValue);
return NBTypeConverter.convertOr(value, defaultValue);
}
}
@ -229,7 +129,6 @@ public abstract class OpTemplate implements Tagged {
}
public <V> V getParam(String name, Class<? extends V> type) {
Object object = getParams().get(name);
if (object == null) {
@ -273,7 +172,7 @@ public abstract class OpTemplate implements Tagged {
*
* @return an optional {@link ParsedStringTemplate}
*/
public Optional<ParsedStringTemplate> getParsed(Function<String,String>... rewriters) {
public Optional<ParsedStringTemplate> getParsed(Function<String, String>... rewriters) {
Optional<String> os = getStmt();
return os.map(s -> {
String result = s;
@ -281,7 +180,7 @@ public abstract class OpTemplate implements Tagged {
result = rewriter.apply(result);
}
return result;
}).map(s -> new ParsedStringTemplate(s,getBindings()));
}).map(s -> new ParsedStringTemplate(s, getBindings()));
}
public Optional<ParsedStringTemplate> getParsed() {
@ -309,7 +208,7 @@ public abstract class OpTemplate implements Tagged {
fields.put(FIELD_TAGS, this.getTags());
}
this.getOp().ifPresent(o -> fields.put(FIELD_OP,o));
this.getOp().ifPresent(o -> fields.put(FIELD_OP, o));
fields.put(FIELD_NAME, this.getName());
@ -320,10 +219,11 @@ public abstract class OpTemplate implements Tagged {
* Legacy support for String form statements. This is left here as a convenience method,
* however it is changed to an Optional to force caller refactorings.
*
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op fields at all.
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op
* fields at all.
*/
public Optional<String> getStmt() {
return getOp().map(m->m.get("stmt")).map(s->{
return getOp().map(m -> m.get("stmt")).map(s -> {
if (s instanceof CharSequence) {
return s.toString();
} else {
@ -333,7 +233,7 @@ public abstract class OpTemplate implements Tagged {
}
public Element getParamReader() {
return NBParams.one(getName(),getParams());
return NBParams.one(getName(), getParams());
}
/**
@ -347,13 +247,13 @@ public abstract class OpTemplate implements Tagged {
* @return the map of all remaining fields from the op template and the params map.
*/
public Map<String, Object> remainingFields() {
Map<String,Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
Map<String, Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
remaining.putAll(getParams());
return remaining;
}
public void assertConsumed() {
if (size()>0) {
if (size() > 0) {
throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields());
}
}

94
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplateFormat.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.yaml;
import java.net.URI;
import java.nio.file.Path;
/**
* The formats which are recognized as source data for the workload. Any serialization may be supported
* which can be converted from a character string to an {@link io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList} structure.
*
* Those which are derived from paths may be determined by their filename extension. Others, which are provided from internal
* NoSQLBench sources, may only be invoked explicitly.
*/
public enum OpTemplateFormat {
/**
* The source is in YAML format
*/
yaml("yaml", "yml"),
/**
* The source is in JSON format
*/
json("json", "json5"),
/**
* The source is in jsonnet format, suitable for pre-processing with context data
*/
jsonnet("jsonnet", "jsonnet5"),
/**
* The source is inline, meaning {@link io.nosqlbench.api.config.params.ParamsParser} format.
* This is similar to the {@link #stmt} format except that it is parsed for internal op structure.
* This format is not detectable by source path, and thus can only be used when provided directly
* from the caller.
*/
inline(),
/**
* The source is in single-statement form, meaning that it is known to be the value of the 'stmt'
* field of an op template. This format is not detectable by source path, and thus can only be used when provided directly
* from the caller.
*/
stmt();
private final String[] pathExtensions;
OpTemplateFormat(String... pathExtensions) {
this.pathExtensions = pathExtensions;
}
public static OpTemplateFormat valueOfURI(URI uri) {
var fullName = uri.toString();
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
for (OpTemplateFormat value : values()) {
for (String pathExtension : value.pathExtensions) {
if (pathExtension.equals(extension)) {
return value;
}
}
}
throw new RuntimeException("Unable to determine source format for " + uri);
}
public static OpTemplateFormat valueOfPath(Path path) {
var fullName = path.toString();
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
for (OpTemplateFormat value : values()) {
for (String pathExtension : value.pathExtensions) {
if (pathExtension.equals(extension)) {
return value;
}
}
}
throw new RuntimeException("Unable to determine source format for " + path);
}
}

44
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsBlock.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsBlock.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,34 +17,34 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock;
import io.nosqlbench.api.engine.util.Tagged;
import org.jetbrains.annotations.NotNull;
import java.util.*;
public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
public class OpsBlock implements Tagged, Iterable<OpTemplate> {
private final RawStmtsBlock rawStmtsBlock;
private final StmtsDoc rawStmtsDoc;
private final RawOpsBlock rawOpsBlock;
private final OpsDoc rawOpsDoc;
private final int blockIdx;
public StmtsBlock(RawStmtsBlock rawStmtsBlock, StmtsDoc rawStmtsDoc, int blockIdx) {
this.rawStmtsBlock = rawStmtsBlock;
this.rawStmtsDoc = rawStmtsDoc;
public OpsBlock(RawOpsBlock rawOpsBlock, OpsDoc rawOpsDoc, int blockIdx) {
this.rawOpsBlock = rawOpsBlock;
this.rawOpsDoc = rawOpsDoc;
this.blockIdx = blockIdx;
}
public List<OpTemplate> getOps() {
List<OpTemplate> rawOpTemplates = new ArrayList<>();
List<RawStmtDef> statements = rawStmtsBlock.getRawStmtDefs();
List<RawOpDef> opDefs = rawOpsBlock.getRawOpDefs();
for (int i = 0; i < statements.size(); i++) {
for (int i = 0; i < opDefs.size(); i++) {
rawOpTemplates.add(
new OpDef(this, statements.get(i))
new OpDef(this, opDefs.get(i))
);
}
return rawOpTemplates;
@ -52,11 +52,11 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
public String getName() {
StringBuilder sb = new StringBuilder();
if (!rawStmtsDoc.getName().isEmpty()) {
sb.append(rawStmtsDoc.getName()).append("--");
if (!rawOpsDoc.getName().isEmpty()) {
sb.append(rawOpsDoc.getName()).append("--");
}
if (!rawStmtsBlock.getName().isEmpty()) {
sb.append(rawStmtsBlock.getName());
if (!rawOpsBlock.getName().isEmpty()) {
sb.append(rawOpsBlock.getName());
} else {
sb.append("block").append(blockIdx);
}
@ -64,15 +64,15 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
}
public Map<String, String> getTags() {
return new MultiMapLookup<>(rawStmtsBlock.getTags(), rawStmtsDoc.getTags());
return new MultiMapLookup<>(rawOpsBlock.getTags(), rawOpsDoc.getTags());
}
public Map<String, Object> getParams() {
return new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
return new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
}
public Map<String, String> getParamsAsText() {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
LinkedHashMap<String, String> stringmap = new LinkedHashMap<>();
lookup.forEach((k, v) -> stringmap.put(k, v.toString()));
return stringmap;
@ -81,7 +81,7 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
@SuppressWarnings("unchecked")
public <V> V getParamOrDefault(String name, V defaultValue) {
Objects.requireNonNull(defaultValue);
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
if (!lookup.containsKey(name)) {
return defaultValue;
}
@ -90,14 +90,14 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
}
public <V> V getParam(String name, Class<? extends V> type) {
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
Object object = lookup.get(name);
V value = type.cast(object);
return value;
}
public Map<String, String> getBindings() {
return new MultiMapLookup<>(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings());
return new MultiMapLookup<>(rawOpsBlock.getBindings(), rawOpsDoc.getBindings());
}
@Override

50
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDoc.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDoc.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +16,8 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc;
import io.nosqlbench.api.engine.util.Tagged;
import java.util.ArrayList;
@ -27,29 +27,29 @@ import java.util.Map;
import java.util.stream.Collectors;
/**
* StmtsDoc creates a logical view of a statements doc that includes
* OpsDoc creates a logical view of a RawOpsDoc doc that includes
* all inherited and overridden values for bindings, tags, and params.
*/
public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
public class OpsDoc implements Tagged, Iterable<OpsBlock> {
private final RawStmtsDoc rawStmtsDoc;
private final RawOpsDoc rawOpsDoc;
public StmtsDoc(RawStmtsDoc rawStmtsDoc) {
this.rawStmtsDoc = rawStmtsDoc;
public OpsDoc(RawOpsDoc rawOpsDoc) {
this.rawOpsDoc = rawOpsDoc;
}
/**
* @return a usable list of blocks, including inherited bindings, params, and tags
* from the parent doc
*/
public List<StmtsBlock> getBlocks() {
List<StmtsBlock> blocks = new ArrayList<>();
public List<OpsBlock> getBlocks() {
List<OpsBlock> blocks = new ArrayList<>();
int blockIdx = 0;
for (RawStmtsBlock rawStmtsBlock : rawStmtsDoc.getBlocks()) {
String compositeName = rawStmtsDoc.getName() +
(rawStmtsBlock.getName().isEmpty() ? "" : "-" + rawStmtsBlock.getName());
StmtsBlock compositeBlock = new StmtsBlock(rawStmtsBlock, this, ++blockIdx);
for (RawOpsBlock rawOpsBlock : rawOpsDoc.getBlocks()) {
String compositeName = rawOpsDoc.getName() +
(rawOpsBlock.getName().isEmpty() ? "" : "-" + rawOpsBlock.getName());
OpsBlock compositeBlock = new OpsBlock(rawOpsBlock, this, ++blockIdx);
blocks.add(compositeBlock);
}
@ -61,53 +61,53 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
*/
@Override
public Map<String, String> getTags() {
return rawStmtsDoc.getTags();
return rawOpsDoc.getTags();
}
/**
* @return a usable map of parameters, including those inherited from the parent doc
*/
public Map<String, Object> getParams() {
return rawStmtsDoc.getParams();
return rawOpsDoc.getParams();
}
/**
* @return a usable map of bindings, including those inherited from the parent doc
*/
public Map<String, String> getBindings() {
return rawStmtsDoc.getBindings();
return rawOpsDoc.getBindings();
}
/**
* @return the name of this block
*/
public String getName() {
return rawStmtsDoc.getName();
return rawOpsDoc.getName();
}
/**
* @return The list of all included statements for all included block in this document,
* including the inherited and overridden values from the this doc and the parent block.
* @return The list of all included op templates for all included block in this document,
* including the inherited and overridden values from this doc and the parent block.
*/
public List<OpTemplate> getStmts() {
public List<OpTemplate> getOpTemplates() {
return getBlocks().stream().flatMap(b -> b.getOps().stream()).collect(Collectors.toList());
}
/**
* Allow StmtsDoc to be used in iterable loops.
* @return An iterator of {@link StmtsBlock}
* @return An iterator of {@link OpsBlock}
*/
@Override
public Iterator<StmtsBlock> iterator() {
public Iterator<OpsBlock> iterator() {
return getBlocks().iterator();
}
public Scenarios getScenarios() {
return new Scenarios(rawStmtsDoc.getRawScenarios());
return new Scenarios(rawOpsDoc.getRawScenarios());
}
public String getDescription() {
return rawStmtsDoc.getDesc();
return rawOpsDoc.getDesc();
}
}

56
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocList.java
View File

@ -16,9 +16,9 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
import io.nosqlbench.engine.api.util.TagFilter;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfigModel;
@ -31,48 +31,48 @@ import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class StmtsDocList implements Iterable<StmtsDoc> {
private final static Logger logger = LogManager.getLogger(StmtsDocList.class);
public class OpsDocList implements Iterable<OpsDoc> {
private final static Logger logger = LogManager.getLogger(OpsDocList.class);
private final RawStmtsDocList rawStmtsDocList;
private final RawOpsDocList rawOpsDocList;
private final Map<String, String> templateVariables = new LinkedHashMap<>();
public StmtsDocList(RawStmtsDocList rawStmtsDocList) {
this.rawStmtsDocList = rawStmtsDocList;
public OpsDocList(RawOpsDocList rawOpsDocList) {
this.rawOpsDocList = rawOpsDocList;
}
public static StmtsDocList none() {
return new StmtsDocList(RawStmtsDocList.none());
public static OpsDocList none() {
return new OpsDocList(RawOpsDocList.none());
}
public List<StmtsDoc> getStmtDocs(String tagFilter) {
public List<OpsDoc> getStmtDocs(String tagFilter) {
TagFilter tf = new TagFilter(tagFilter);
return getStmtDocs().stream()
.filter(tf::matchesTagged)
.collect(Collectors.toList());
}
public List<StmtsDoc> getStmtDocs() {
return rawStmtsDocList.getStmtsDocs().stream()
.map(StmtsDoc::new)
public List<OpsDoc> getStmtDocs() {
return rawOpsDocList.getOpsDocs().stream()
.map(OpsDoc::new)
.collect(Collectors.toList());
}
public List<OpTemplate> getStmts() {
return getStmts("");
public List<OpTemplate> getOps() {
return getOps("");
}
/**
* @param tagFilterSpec a comma-separated tag filter spec
* @return The list of all included statements for all included blocks of in this document,
* including the inherited and overridden values from the this doc and the parent block.
* @return The list of all included op templates for all included blocks of in this document,
* including the inherited and overridden values from this doc and the parent block.
*/
public List<OpTemplate> getStmts(String tagFilterSpec) {
public List<OpTemplate> getOps(String tagFilterSpec) {
TagFilter ts = new TagFilter(tagFilterSpec);
List<OpTemplate> opTemplates = new ArrayList<>();
getStmtDocs().stream()
.flatMap(d -> d.getStmts().stream())
.flatMap(d -> d.getOpTemplates().stream())
.filter(ts::matchesTagged)
.forEach(opTemplates::add);
@ -81,7 +81,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
@Override
public Iterator<StmtsDoc> iterator() {
public Iterator<OpsDoc> iterator() {
return getStmtDocs().iterator();
}
@ -94,7 +94,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
public Map<String, String> getDocBindings() {
LinkedHashMap<String, String> docBindings = new LinkedHashMap<>();
getStmtDocs().stream()
.map(StmtsDoc::getBindings)
.map(OpsDoc::getBindings)
.forEach(docBindings::putAll);
return docBindings;
}
@ -130,7 +130,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
}
public NBConfigModel getConfigModel() {
ConfigModel cfgmodel = ConfigModel.of(StmtsDocList.class);
ConfigModel cfgmodel = ConfigModel.of(OpsDocList.class);
getTemplateVariables().forEach((k, v) -> {
cfgmodel.add(Param.defaultTo(k, v, "template parameter found in the yaml workload"));
});
@ -144,9 +144,9 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
int blockscount = 0;
int opscount = 0;
for (StmtsDoc stmtDoc : this.getStmtDocs()) {
for (OpsDoc stmtDoc : this.getStmtDocs()) {
docscount++;
for (StmtsBlock block : stmtDoc.getBlocks()) {
for (OpsBlock block : stmtDoc.getBlocks()) {
blockscount++;
for (OpTemplate op : block.getOps()) {
opscount++;
@ -155,18 +155,16 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
}
sb.append("docs: " + docscount + " blocks:" + blockscount + " ops:" + opscount);
// String names = this.rawStmtsDocList.getStmtsDocs().stream().flatMap(sd -> sd.getRawStmtDefs().stream()).map(d->d.getName()).collect(Collectors.joining(","));
// sb.append(", names:").append(names);
return sb.toString();
}
public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> {
StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities");
OpsDocList loaded = OpsLoader.loadPath((String) workload, Map.of(), "activities");
return loaded.getConfigModel();
};
public Pattern getVersionRegex() {
List<RawStmtsDoc> stmtDocs = rawStmtsDocList.getStmtsDocs();
List<RawOpsDoc> stmtDocs = rawOpsDocList.getOpsDocs();
return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*");
}
}

36
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverSpaceCache.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -31,34 +31,24 @@ import java.util.function.Function;
* a client API for advanced testing. The default instance should simply be named
* {@code default}</p>
*
* <p>Most native drivers use some combination of fluent, functional, and declarative
* patterns. These usually require you to keep access to a set of core
* state-holding objects in order to construct new elements to drive operations with.
* An example of this would be creating a statement from a session. It is necessary
* to keep the session around in for when you create new statements. Maintaining
* the session object is considered an essential part of idiomatic and efficient
* use of the API. Further, you may have builders or factories that are created
* from the session which should be cached as well. Keeping all these objects
* together requires attaching them to a cohesive owning object -- That is the space
* cache.</p>
* <p>Most native drivers use some combination of fluent, functional, and declarative patterns. These usually require
* you to keep access to a set of core state-holding objects in order to construct new elements to drive operations
* with. An example of this would be creating an executable operation from a session object. It is necessary to keep the
* session around in for when you create new statements. Maintaining the session object is considered an essential part
* of idiomatic and efficient use of the API. Further, you may have builders or factories that are created from the
* session which should be cached as well. Keeping all these objects together requires attaching them to a cohesive
* owning object -- That is the space cache.</p>
*
* <p>You might want to create multiple session contexts in order to test out
* non-trivial behavior in advanced testing scenarios. To do this dynamically, i.e.
* acquire some named space cache, simply call the {@link #get(String)}</p> method
* with the name of the space you want to use. This value can be provided as a
* dynamic field in your op mapping ({@link io.nosqlbench.engine.api.activityimpl.OpMapper})
* or synthesis ({@link io.nosqlbench.engine.api.activityimpl.OpDispenser})
* implementation.
*
* @param <S> The type which will represent the cache for a given type of adapter.
* @param <S>
* The type which will represent the cache for a given type of adapter.
*/
public class DriverSpaceCache<S> {
private final ConcurrentHashMap<String,S> cache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, S> cache = new ConcurrentHashMap<>();
private final Function<String, S> newSpaceFunction;
public DriverSpaceCache(Function<String,S> newSpaceFunction) {
public DriverSpaceCache(Function<String, S> newSpaceFunction) {
this.newSpaceFunction = newSpaceFunction;
}
@ -66,7 +56,7 @@ public class DriverSpaceCache<S> {
return cache.computeIfAbsent(name, newSpaceFunction);
}
public Map<String,S> getElements() {
public Map<String, S> getElements() {
return Collections.unmodifiableMap(cache);
}

29
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/decorators/SyntheticOpTemplateProvider.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,29 +17,34 @@
package io.nosqlbench.engine.api.activityimpl.uniform.decorators;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.templating.DriverAdapterDecorators;
import java.util.List;
import java.util.Map;
/**
* In the case that no statements are provided for an activity, but a workload
* description is provided, and the result for no found statements is not
* simply a matter of tag filtering, an activity may provide its own
* synthetic ops. This is here primarily to support the classic behavior of stdout
* until it's functionality is subsumed by standard diagnostic features.
*
* Note that this is only valid while an activity uses a single driver, which will
* change with upcoming API updates.
* It is possible for a DriverAdapter to create op templates using partial information.
* For example, the stdout driver can use only bindings to create CSV or JSON style data renderings.
* This mechanism is only triggered when:
* <OL>
* <LI>No op templates were provided.</LI>
* <LI>The default driver for an activity implements this method.</LI>
* </OL>
* <p>
* This excludes cases where a workload was provided with op templates, but they were all filtered out. In that case,
* the user should be informed of an error.
*/
public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators {
/**
* If a driver adapter supports creating example op templates from bindings,
* it must implement this method to do so.
* @param stmtsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings
*
* @param opsDocList
* The existing doc structure, which should contain no fully defined op templates, but may contain other
* elements like bindings
* @return A list of op templates, size zero or more
*/
List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> params);
List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params);
}

34
adapters-api/src/main/java/io/nosqlbench/engine/api/templating/OpTemplateSupplier.java
View File

@ -1,34 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.api.config.standard.NBConfiguration;
import java.util.List;
import java.util.Optional;
/**
* An Op Template Supplier can provide its own source of op templates instead
* of relying on the built-in mechanism. By default, the built-in mechanism
* will read op definitions from parameters first, then any ops (statements)
* from yaml files provided in the workload= activity parameters.
*/
public interface OpTemplateSupplier extends DriverAdapterDecorators {
Optional<List<OpTemplate>> loadOpTemplates(NBConfiguration cfg);
}

42
adapters-api/src/main/resources/workload_definition/00_workload_specification.md Normal file
View File

@ -0,0 +1,42 @@
# Workload Specification
This directory contains the testable specification for workload definitions used by NoSQLBench.
All the content blocks in this section have been validated with the latest NoSQLBench build.
Usually, users will not need to delve too deeply into this section. It is useful as a detailed
guide for contributors and driver developers. If you are using a driver which leaves you
wondering what a good op template example looks like, then the driver needs better examples in
its documentation!
# Synopsis
There are two primary views of workload definitions that we care about:
1. The User View of **op templates**
1. Op templates are simply the schematic recipes for building an operation once you know the
cycle it is for.
2. Op templates are provided by users in YAML or JSON or even directly via runtime API. This
is called a workload template, which contains op templates.
3. Op templates can be provided with optional metadata which serve to label, group,
parameterize or otherwise make the individual op templates more manageable.
4. A variety of forms are supported which are self-evident, but which allow users to have
some flexibility in how they structure their YAML, JSON, or runtime collections. **This
specification is about how these various forms are allowed, and how they relate to a
fully-qualified and de-normalized op template view.
2. The Developer View of the ParsedOp API. This is the view of an op template which presents the
developer with a very high-level toolkit for building op synthesis functions.
# Details
The documentation in this directory serve as a testable specification for all the above. It
shows specific examples of all the valid op template forms in both YAML and JSON, as well as how
the data is normalized to feed developer's view of the ParsedOp API.
## Related Reading
If you want to understand the rest of this document, it is crucial that you have a working knowledge
of the standard YAML format and several examples from the current drivers. You can learn this from
the main documentation which demonstrates step-by-step how to build a workload. Reading further in
this document will be most useful for core NB developers, or advanced users who want to know all
the possible ways of building workloads.

0
adapters-api/src/main/resources/workload_definition/spectest_formatting.md → adapters-api/src/main/resources/workload_definition/01_spectest_formatting.md
View File

0
adapters-api/src/main/resources/workload_definition/workload_structure.md → adapters-api/src/main/resources/workload_definition/02_workload_structure.md
View File

0
adapters-api/src/main/resources/workload_definition/op_template_basics.md → adapters-api/src/main/resources/workload_definition/04_op_template_basics.md
View File

0
adapters-api/src/main/resources/workload_definition/op_template_payloads.md → adapters-api/src/main/resources/workload_definition/05_op_template_payloads.md
View File

0
adapters-api/src/main/resources/workload_definition/op_template_variations.md → adapters-api/src/main/resources/workload_definition/06_op_template_variations.md
View File

0
adapters-api/src/main/resources/workload_definition/template_variables.md → adapters-api/src/main/resources/workload_definition/07_template_variables.md
View File

205
adapters-api/src/main/resources/workload_definition/08_parsed_op_api.md Normal file
View File

@ -0,0 +1,205 @@
# ParsedOp API
In the workload template examples, we show statements as being formed from a string value. This is a
specific type of statement form, although it is possible to provide structured op templates as well.
**The ParsedOp API is responsible for converting all valid op template forms into a consistent and
unambiguous model.** Thus, the rules for mapping the various forms to the command model must be
precise. Those rules are the substance of this specification.
## Op Synthesis
Executable operations are _created_ on the fly by NoSQLBench via a process called _Op Synthesis_.
This is done incrementally in stages. The following narrative describes this process in logical
stages. (The implementation may vary from this, but it explains the effects, nonetheless.)
Everything here happens **during** activity initialization, before the activity starts running
cycles:
1. *Template Variable Expansion* - If there are template variables, such as
`TEMPLATE(name,defaultval)` or `&lt;&lt;name:defaultval&gt;&gt;`, then these are expanded
according to their defaults and any overrides provided in the activity params. This is a macro
substitution only, so the values are simply interposed into the character stream of the document.
2. *Jsonnet Evaluation* - If the source file was in jsonnet format (the extension was `.jsonnet`)
then it is interpreted by sjsonnet, with all activity parameters available as external variables.
3. *Structural Normalization* - The workload template (yaml, json, or data structure) is loaded
into memory and transformed into a standard format. This means taking various list and map
forms at every level and converting them to a singular standard form in memory.
4. *Auto-Naming* - All elements which do not already have a name are assigned a simple name like
`block2` or `op3`.
5. *Auto-Tagging* - All op templates are given standard tag values under reserved tag names:
- **block**: the name of the block containing the op template. For example: `block2`.
- **name**: the name of the op template, prefixed with the block value and `--`. For example,
`block2--op1`.
6. *Property De-normalization* - Default values for all the standard op template properties are
copied from the doc to the block layer unless the same-named key exists. Then the same
method is applied from the doc layer to the op template layer. **At this point, the op
templates are effectively an ordered list of data structures, each containing all necessary
details for use.**
7. *Tag Filtering* - The activity's `tag` param is used to filter all the op templates
according to their tag map.
8. *Bind Point and Capture Points* - Each op template is now converted into a ParsedOp, which is
a swiss-army knife of op template introspection and function generation. It is the direct
programmatic API that driver adapters use in subsequent steps.
- Any string sequences with bind points like `this has a {bindpoint}` are automatically
converted to a long -&gt; string function.
- Any direct references with no surrounding text like `{bindpoint}` are automatically
converted to direct binding references.
- Any other string form is cached as a static value.
- The same process is applied to Lists and Maps, allowing structural templates which read
like JSON with bind points in arbitrary places.
8. *Op Mapping* - Using the ParsedOp API, each op template is categorized by the active `driver`
according to that driver's documented examples and type-matching rules. Once the op mapper
determines what op type a user intended, it uses this information and the associated op
fields to create an *Op Dispenser*.
9. *Op Sequencing* - The op dispensers are kept as an internal sequence, and installed into a
[LUT](https://en.wikipedia.org/wiki/Lookup_table) according to their ratios and the specified
(or default) sequencer. By default, round-robin with bucket exhaustion is used. The ratios
specified are used directly in the LUT.
When this is complete, you are left with an efficient lookup table which indexes into a set of
OpDispensers. The length of this lookup table is called the _sequence length_, and that value is
used, by default, to set the _stride_ for the activity. This stride determines the size of
per-thread cycle batching, effectively turning each sequence into a thread-safe set of
operations which are serialized, and thus suitable for testing linearized operations with
suitable dependency and error-handling mechanisms. (But wait, there's more!)
## Special Cases
Drivers are assigned to op templates individually, meaning you can specify the driver within an
op template, not even assigning a default for the activity. Further, certain drivers are able to
fill in missing details for op templates, like the `stdout` driver which only requires bindings.
This means that there are distinct cases for configuration which are valid, and these are
checked at initialization time:
- A `driver` must be selected for each op template either directly or via activity params.
- If the whole workload template provided does not include actual op templates **AND** a
default driver is provided which can create synthetic op templates, it is given the raw
workload template, incomplete as it is, and asked to provide op templates which have all
normalization, naming, etc. already done. This is injected before the tag-filtering phase.
- In any case that an actual non-zero list of op templates is provided and tag filtering removes
them all, an error is thrown.
- If, after tag filtering no op template are in the active list, an error is thrown.
# The ParsedOp
The components of a fully-parsed op template (AKA a ParsedOp) are:
## name
Each ParsedOp knows its name, which is simply the op template name that it was made from. This
is useful for diagnostics, logging, and metrics.
## description
Every named element of a workload may be given a description.
## tags
Every op template has tags, even if they are auto-assigned from the block and op template names.
If you assign explicit tags to an op template, the standard tags are still provided. Thus, it is
an error to directly provide a tag named `block` or `name`.
## bindings
Although bindings are usually defined as workload template level property, they can also be
provided directly as an op field property.
## op fields
The **op** property of an op template or ParsedOp is the root of the op fields. This is a map of
specific fields specified by the user.
### static op fields
Some op fields are simply static values. Since these values are not generated per cycle, they are
kept separate as reference data. Knowing which fields are static and which are not makes it
possible for developers to optimize op synthesis.
### dynamic op fields
Other fields may be specified as recipes, with the actual value to be filled-in once the cycle
value is known. All such fields are known as _dynamic op fields_, and are provided to the op
dispenser as a long function, where the input is always the cycle value and the output is a
type-specific value as determined by the associated binding recipe.
### bind points
This is how dynamic values are indicated. Each bind point in an op template results in some type of
procedural generation binding. These can be references to named bindings elsewhere in the
workload template, or they can be inline.
### capture points
Names of result values to save, and the variable names they are to be saved as. The names represent
the name as it would be found in the native driver's API, such as the name `userid`
in `select userid from ...`. In string form statements, users can specify that the userid should be
saved as the thread-local variable named *userid* simply by tagging it
like `select [userid] from ...`. They can also specify that this value should be captured under a
different name with a variation like `select [userid as user_id] from ...`. This is the standard
variable capture syntax for any string-based statement form.
### params
A backwards-compatible feature called op params is still available. This is another root
property within an op template which can be used to accessorize op fields. By default, any op
field which is not explicitly rooted under the `op` property are put there anyway. This is also
true when there is an explicitly `params` property. However if the op property is provided, then
all non-reserved fields are given to the params property instead. If both the `op` and the
`param` op properties are specified, then no non-reserved op fields are allowed outside of these
root values. Thus it is possible to still support params, but it is **highly** recommended that
new driver developers avoid using this field, and instead allow all fields to be automatically
anchored under the `op` property. This keeps configs terse and simple going forward.
Params may not be dynamic.
# Mapping Rules
A ParsedOp does not necessarily describe a specific low-level operation to be performed by
a native driver. It *should* do so, but it is up to the user to provide a valid op template
according to the documented rules of op construction for that driver type. These rules should be
clearly documented by the driver developer as examples in markdown that is required for every
driver. With this documentation, users can use `nb5 help &lt;driver&gt;` to see exactly how
to create op templates for a given driver.
## String Form
Basic operations are made from a statement in some type of query language:
```yaml
ops:
- stringform: select [userid] from db.users where user='{username}';
bindings:
username: NumberNameToString()
```
# Reserved op fields
The property names `ratio`, `driver`, `space`, are considered reserved by the NoSQLBench runtime.
These are extracted and handled specially by the core runtime.
# Base OpDispenser fields
The BaseOpDispenser, which <s>is</s> will be required as the base implementation of any op
dispenser going forward, provides cross-cutting functionality. These include `start-timers`,
`stop-timers`, `instrument`, and likely will include more as future cross-driver functionality is
added. These fields will be considered reserved property names.
# Optimization
It should be noted that the op mapping process, where user intentions are mapped from op templates to
op dispensers is not something that needs to be done quickly. This occurs at _initialization_
time. Instead, it is more important to focus on user experience factors, such as flexibility,
obviousness, robustness, correctness, and so on. Thus, priority of design factors in this part
of NB is placed more on clear and purposeful abstractions and less on optimizing for speed. The
clarity and detail which is conveyed by this layer to the driver developer will then enable
them to focus on building fast and correct op dispensers. These dispensers are also constructed
before the workload starts running, but are used at high speed while the workload is running.
In essence:
- Any initialization code which happens before or in the OpDispenser constructor should not be
concerned with careful performance optimization.
- Any code which occurs within the OpDispenser#apply method should be as lightweight as is
reasonable.

99
adapters-api/src/main/resources/workload_definition/README.md
View File

@ -1,99 +0,0 @@
# Workload Specification
This directory contains the testable specification for workload definitions used by NoSQLBench.
## Op Templates vs Developer API
There are two primary views of workload definitions that we care about:
1. The User View of **op templates**
1. Op templates are simply the schematic recipes for building an operation.
2. Op templates are provided by users in YAML or JSON or even directly via runtime API.
3. Op templates can be provided with optional metadata which serves to label, group or
otherwise make the individual op templates more manageable.
4. A variety of forms are supported which are self-evident, but which allow users to have
some flexibility in how they structure their YAML, JSON, or runtime collections.
2. The Developer View of the ParsedOp API -- All op templates, regardless of the form they are
provided in, are processed into a normalized internal data structure.
1. The detailed documentation for the ParsedOp API is in javadoc.
The documentation in this directory serve as a testable specification for all the above. It
shows specific examples of all the valid op template forms in both YAML and JSON, as well as how
the data is normalized to feed developer's view of the ParsedOp API.
If you are a new user, it is recommended that you read the basic docs first before delving into
these specification-level docs too much. The intro docs show normative and simple ways to
specific workloads without worrying too much about all the possible forms.
## Templating Language
When users want to specify a set of operations to perform, they do so with the workload templating
format, which includes document level details, block level details, and op level details.
Specific reserved words like `block` or `ops` are used in tandem with nesting structure to
define all valid workload constructions. Because of this, workload definitions are
essentially data structures comprised of basic collection types and primitive values. Any on-disk
format which can be loaded as such can be a valid source of workload definitions.
- [SpecTest Formatting](spectest_formatting.md) - A primer on the example formats used here
- [Workload Structure](workload_structure.md) - Overall workload structure, keywords, nesting
features
- [Op Template Basics](op_template_basics.md) - Basic Details of op templating
- [Op Template Variations](op_template_variations.md) - Additional op template variants
and corner cases
- [Template Variables](template_variables.md) - Textual macros and default values
## ParsedOp API
After a workload template is loaded into an activity, it is presented to the driver in an API which
is suitable for building executable ops in the native driver.
- [ParsedOp API](parsed_op_api.md) - Defines the API which developers see after a workload is fully
loaded.
## Related Reading
If you want to understand the rest of this document, it is crucial that you have a working knowledge
of the standard YAML format and several examples from the current drivers. You can learn this from
the main documentation which demonstrates step-by-step how to build a workload. Reading further in
this document will be most useful for core NB developers, or advanced users who want to know all
the possible ways of building workloads.
## Op Mapping Stages
The process of loading a workload definition occurs in several discrete steps during a NoSQLBench
session:
1. The workload file is loaded.
2. Template variables from the activity parameters are interposed into the raw contents of the
file.
3. The file is deserialized from its native form into a raw data structure.
4. The raw data structure is transformed into a normalized data structure according to the Op
Template normalization rules.
5. Each op template is then denormalized as a self-contained data
structure, containing all the provided bindings, params, and tags from the upper layers of the
doc structure.
6. The data is provided to the ParsedOp API for use by the developer.
7. The DriverAdapter is loaded which understands the op fields provided in the op template.
8. The DriverAdapter uses its documented rules to determine which types of native driver operations
each op template is intended to represent. This is called **Op Mapping**.
9. The DriverAdapter (via the selected Op Mapper) uses the identified types to create dispensers of
native driver operations. This is called **Op Dispensing**.
10. The op dispensers are arranged into an indexed bank of op sources according to the specified
ratios and or sequencing strategy. From this point on, NoSQLBench has the ability to
construct an operation for any given cycle at high speed.
These specifications are focused on steps 2-5. The DriverAdapter focuses on the developer's use of
the ParsedOp API, and as such is documented in javadoc primarily. Some details on the ParsedOp
API are shared here for basic awareness, but developers should look to the javadoc for the full
story.
## Mapping vs Running
It should be noted that the Op Mapping stage, where user intentions are mapped from op templates to
native operations is not something that needs to be done quickly. This occurs at
_initialization_ time. Instead, it is more important to focus on user experience factors, such as
flexibility, obviousness, robustness, correctness, and so on. Thus, priority of design factors in
this part of NB is placed more on clear and purposeful abstractions and less on optimizing for
speed. The clarity and detail which is conveyed by this layer to the driver developer will then
enable them to focus on building fast and correct op dispensers. These dispensers are also
constructed before the workload starts running, but are used at high speed while the workload
is running.

301
adapters-api/src/main/resources/workload_definition/parsed_op_api.md
View File

@ -1,301 +0,0 @@
# ParsedOp API
In the workload template examples, we show statements as being formed from a string value. This is a
specific type of statement form, although it is possible to provide structured op templates as well.
**The ParsedOp API is responsible for converting all valid op template forms into a consistent and
unambiguous model.** Thus, the rules for mapping the various forms to the command model must be
precise. Those rules are the substance of this specification.
## Op Synthesis
The method of turning an op template, some data generation functions, and some seed values into an
executable operation is called *Op Synthesis* in NoSQLBench. This is done in incremental stages:
1. During activity initialization, NoSQLBench parses the workload template and op templates
contained within. Each active op template (after filtering) is converted to a parsed command.
2. The NB driver uses the parsed command to guide the construction of an OpDispenser<T>. This is a
dispenser of operations that can be executed by the driver's Action implementation.
3. When it is time to create an actual operation to be executed, unique with its own procedurally
generated payload and settings, the OpDispenser<T> is invoked as a LongFunction<T>. The input
provided to this function is the cycle number of the operation. This is essentially a seed that
determines the content of all the dynamic fields in the operation.
This process is non-trivial in that it is an incremental creational pattern, where the resultant
object is contextual to some native API. The command API is intended to guide and enable op
synthesis without tying developers' hands.
## Command Fields
A command structure is intended to provide all the fields needed to fully realize a native
operation. Some of these fields will be constant, or *static* in the op template, expressed simply
as strings, numbers, lists or maps. These are parsed from the op template as such and are cached in
the command structure as statics.
Other fields are only prescribed as recipes. This comes in two parts: 1) The description for how to
create the value from a binding function, and 2) the binding point within the op template. Suppose
you have a string-based op template like this:
```yaml
ops:
- op1: select * from users where userid={userid}
bindings:
userid: ToString();
```
In this case, there is only one op in the list of ops, having a name `op1` and a string form op
template of `select * from users where userid={userid}`.
## Parsed Command Structure
Once an op template is parsed into a *parsed command*, it has the state shown in the data structure
schematic below:
```json
{
"name": "some-map-name",
"statics": {
"s1": "v1",
"s2": {
"f1": "valfoo"
}
},
"dynamics": {
"d1": "NumberNameToString()"
},
"captures": {
"resultprop1": "asname1"
}
}
```
If either an **op** or **stmt** field is provided, then the same structure as above is used:
```json
{
"name": "some-string-op",
"statics": {
},
"dynamics": {
"op": "select username from table where name userid={userid}"
},
"captures": {
}
}
```
The parts of a parsed command structure are:
### command name
Each command knows its name, just like an op template does. This can be useful for diagnostics and
metric naming.
### static fields
The field names which are statically assigned and their values of any type. Since these values are
not generated per-op, they are kept separate as reference data. Knowing which fields are static and
which are not makes it possible for developers to optimize op synthesis.
### dynamic fields
Named bindings points within the op template. These values will only be known for a given cycle.
### variable captures
Names of result values to save, and the variable names they are to be saved as. The names represent
the name as it would be found in the native driver's API, such as the name `userid`
in `select userid from ...`. In string form statements, users can specify that the userid should be
saved as the thread-local variable named *userid* simply by tagging it
like `select [userid] from ...`. They can also specify that this value should be captured under a
different name with a variation like `select [userid as user_id] from ...`. This is the standard
variable capture syntax for any string-based statement form.
# Resolved Command Structure
Once an op template has been parsed into a command structure, the runtime has everything it needs to
know in order to realize a specific set of field values, *given a cycle number*. Within a cycle, the
cycle number is effectively a seed value that drives the generation of all dynamic data for that
cycle.
However, this seed value is only known by the runtime once it is time to execute a specific cycle.
Thus, it is the developer's job to tell the NoSQLBench runtime how to map from the parsed structure
to a native type of executable operation suitable for execution with that driver.
# Interpretation
A command structure does not necessarily describe a specific low-level operation to be performed by
a native driver. It *should* do so, but it is up to the user to provide a valid op template
according to the documented rules of op construction for that driver type. These rules should be
clearly documented by the driver developer.
Once the command structure is provided, the driver takes over and maps the fields into an executable
op -- *almost*. In fact, the driver developer defines the ways that a command structure can be
turned into an executable operation. This is expressed as a *Function<CommandTemplate,T>* where T is
the type used in the native driver's API.
How a developer maps a structure like the above to an operations is up to them. The general rule of
thumb is to use the most obvious and familiar representation of an operation as it would appear to a
user. If this is CQL or SQL, then recommend use that as the statement form. If it GraphQL, use that.
In both of these cases, you have access to
## String Form
Basic operations are made from a statement in some type of query language:
```yaml
ops:
- stringform: select [userid] from db.users where user='{username}';
bindings:
username: NumberNameToString()
```
## Structured Form
Some operations can't be easily represented by a single statement. Some operations are built from a
set of fields which describe more about an operation than the basic statement form. These types of
operations are expressed to NoSQLBench in map or *object* form, where the fields within the op can
be specified independently.
```yaml
ops:
- structured1:
stmt: select * from db.users where user='{username}}';
prepared: true
consistency_level: LOCAL_QUORUM
bindings:
username: NumberNameToString();
- structured2:
cmdtype: "put"
namespace: users
key: { userkey }
body: "User42 was here"
bindings:
userkey: FixedValue(42)
```
In the first case, the op named *structured1* is provided as a string value within a map structure.
The *stmt* field is a reserved word (synonomous with op and operation). When you are reading an op
from the command API, these will represented in exactly the same way as the stringform example
above.
In the second case,
In the second, the op named *structured form* is provided as a map. Both of these examples would
make sense to a user, as they are fairly self-explanatory.
Op templates may specify an op as either a string or a map. No other types are allowed. However,
there are no restrictions placed on the elements below a map.
The driver developer should not have to parse all the possible structural forms that users can
provide. There should be one way to access all of these in a consistent and unambiguous API.
## Command Structure
Here is an example data structure which illustrates all the possible elements of a parsed command:
```json
{
"statics": {
"prepared": "true",
"consistency_level'"
}
}
```
Users provide a template form of an operation in each op template. This contains a sketch of what an
operation might look like, and includes the following optional parts:
- properties of the operation, whether meta (like a statement) or payload content
- the binding points where generated field values will be injected
- The names of values to be extracted from the result of a successful operation.
## Statement Forms
Sometimes operations are derived from a query language, and are thus self-contained in a string
form.
When mapping the template of an operation provided by users to an executable operation in some
native driver with specific values, you have to know
* The s
* The substance of the operation: The name and values of the fields that the user provides as part
of the operation
Command templates are the third layer of workload templating. As described in other spec documents,
the other layers are:
1. [Workload level templates](templated_workloads.md) - This specification covers the basics of a
workload template, including the valid properties and structure.
2. [Operation level templates](templated_operations.md) - This specification covers how operations
can be specified, including semantics and structure.
3. Command level templates, explained below. These are the detailed views of what goes into an op
template, parsed and structured in a way that allows for efficient use at runtime.
Users do not create command templates directly. Instead, these are the *parsed* form of op templates
as seen by the NB driver. The whole point of a command template is to provide crisp semantics and
structure about what a user is asking a driver to do. Command Template
Command templates are essentially schematics for an operation. They are a structural interpretation
of the content provided by users in op templates. Each op template provided can be converted into a
command template. In short, the op template is the form that users tend to edit in yaml or provided
as a data structure via scripting. **Command templates are the view of an op template as seen by an
NB driver.**
```
### Command Templates
Command templates are part of the workload API.
There exists a need to provide op templates to a myriad of runtime APIs,
and thus it has to be flexible enough to serve them all.
1. In some cases, an operation is based on a query language where the
query language itself encodes everything needed for specific operation.
SQL queries are like this. This is a nice simplification, but it is not
realistic for systems build on modern distributed principles.
2. In most cases, you have both an operation and some qualifying rules
about how the operation should be handled, such as consistency level.
Thus, there is a need to provide parameters which can decorate
operations.
3. In some cases, you have a payload for your operation which is not based
on a query language, but instead on an object with fields, or a verb
which determines what other fields are needed. This structure is better
described as a *command* than a *statement*.
4. Finally, you must support separate both of the latter cases where the
command or operations is defined in some pseudo-structured way, but it
also has *separately* a set of qualifying parameters which are
considered orthogonal, or at least separate from the meaning of the
operation itself.
To address the full set of these mapping requirements, a type has been
added to NB which provides a structured and pre-baked version of a
resolvable command -- the CommandTemplate.
This type provides a view to the driver builder of all the fields
specified by the user, whether as encoded as a string, such
as `select row from ...`, or by a set of properties such
as `{"verb":"get",
"id":"2343"}`. It also exposes the parameters separately if provided.
### Static vs Dynamic command fields
Further, for each field in the command template, the driver implementor
knows whether this was provided as a static value or one that can only be
realized for a specific cycle (seed data). Thus, it is possible for
advanced op mapping implementations to optimize the way that new
operations are synthesized for efficiency.
For example, if you know that you have a command which has no dynamic
fields in its command template, then it is possible to create a singleton
op template which can simply be re-used. A fully dynamic command template,
in contrast, may need to be realized dynamically for each cycle, given
that you don't know the value of the fields in the command until you know
the cycle value.
```

20
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/StatementsLoaderTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +16,8 @@
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.junit.jupiter.api.Test;
import java.util.List;
@ -25,16 +25,16 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StatementsLoaderTest {
public class OpsLoaderTest {
@Test
public void testTemplateVarSubstitution() {
StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources");
assertThat(stmtsDocs).isNotNull();
List<StmtsDoc> docs = stmtsDocs.getStmtDocs();
OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
assertThat(opsDocs).isNotNull();
List<OpsDoc> docs = opsDocs.getStmtDocs();
assertThat(docs).hasSize(1);
StmtsDoc stmtsBlocks = docs.get(0);
Map<String, String> bindings = stmtsBlocks.getBindings();
OpsDoc opsDoc = docs.get(0);
Map<String, String> bindings = opsDoc.getBindings();
assertThat(bindings).isEqualTo(Map.of(
"b1a","Prefix(\"prefix\")",
"b1b","Prefix(\"prefix\")",
@ -46,7 +46,7 @@ public class StatementsLoaderTest {
public void testInvalidYamlProperties() {
Exception caught = null;
try {
StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources");
OpsLoader.loadPath("activities/invalid_prop", Map.of(),"src/test/resources");
} catch (Exception e) {
caught = e;
}

60
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpDefTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,11 +16,11 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsBlock;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -37,22 +37,22 @@ public class OpDefTest {
@Test
public void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
OpsDoc doc1 = all.getStmtDocs().get(0);
assertThat(doc1.getName()).isEqualTo("doc1");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsDoc doc2 = all.getStmtDocs().get(1);
OpsDoc doc2 = all.getStmtDocs().get(1);
assertThat(doc2.getBlocks()).hasSize(2);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getBindings()).hasSize(2);
assertThat(block1.getName()).isEqualTo("doc1--block0");
assertThat(block1.getTags()).hasSize(1);
StmtsBlock block21 = doc2.getBlocks().get(0);
StmtsBlock block22 = doc2.getBlocks().get(1);
OpsBlock block21 = doc2.getBlocks().get(0);
OpsBlock block22 = doc2.getBlocks().get(1);
assertThat(block21.getName()).isEqualTo("doc2--block1");
assertThat(block21.getTags()).hasSize(3);
@ -64,11 +64,11 @@ public class OpDefTest {
@Test
public void testStatementRendering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = all.getStmtDocs().get(0);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsDoc doc1 = all.getStmtDocs().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("doc1--block0");
List<OpTemplate> ops = block1.getOps();
assertThat(ops).hasSize(2);
@ -79,10 +79,10 @@ public class OpDefTest {
@Test
public void testConsumableMapState() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getStmts();
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc block1 = docs.get(1);
List<OpTemplate> stmts = block1.getOpTemplates();
OpTemplate stmt0 = stmts.get(0);
OpTemplate stmt1 = stmts.get(1);
assertThat(stmt0.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz"));
@ -94,12 +94,12 @@ public class OpDefTest {
@Test
public void testMapOfMaps() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc0 = docs.get(0);
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc0 = docs.get(0);
assertThat(doc0.getName()).isEqualTo("map-of-maps");
assertThat(doc0.getBlocks()).hasSize(1);
StmtsBlock block1 = doc0.getBlocks().get(0);
OpsBlock block1 = doc0.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("map-of-maps--block0");
assertThat(block1.getOps()).hasSize(2);
OpTemplate op0 = block1.getOps().get(0);
@ -115,12 +115,12 @@ public class OpDefTest {
@Test
public void testBasicStringStmt() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc1 = docs.get(1);
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc1 = docs.get(1);
assertThat(doc1.getName()).isEqualTo("string-statement");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getName()).isEqualTo("string-statement--block0");
assertThat(block1.getOps()).hasSize(1);
OpTemplate op0 = block1.getOps().get(0);
@ -130,12 +130,12 @@ public class OpDefTest {
@Test
public void testListOfNamedMap() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
List<StmtsDoc> docs = all.getStmtDocs();
StmtsDoc doc2 = docs.get(2);
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
List<OpsDoc> docs = all.getStmtDocs();
OpsDoc doc2 = docs.get(2);
assertThat(doc2.getName()).isEqualTo("list-of-named-map");
assertThat(doc2.getBlocks()).hasSize(1);
StmtsBlock block1 = doc2.getBlocks().get(0);
OpsBlock block1 = doc2.getBlocks().get(0);
assertThat(block1.getOps()).hasSize(1);
OpTemplate op0 = block1.getOps().get(0);
System.out.println(op0.getParams());

15
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtEscapingTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpEscapingTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.*;
import org.junit.jupiter.api.BeforeAll;
import org.apache.logging.log4j.Logger;
@ -24,26 +24,27 @@ import org.apache.logging.log4j.LogManager;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StmtEscapingTest {
public class OpEscapingTest {
private final static Logger logger = LogManager.getLogger(StmtEscapingTest.class);
private final static Logger logger = LogManager.getLogger(OpEscapingTest.class);
private static List<OpTemplate> defs;
@BeforeAll
public static void testLayering() {
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml");
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_ops.yaml", Map.of());
assertThat(all).isNotNull();
assertThat(all.getStmtDocs()).hasSize(1);
StmtsDoc doc1 = all.getStmtDocs().get(0);
OpsDoc doc1 = all.getStmtDocs().get(0);
// assertThat(doc1.getName()).isEqualTo("doc1");
assertThat(doc1.getBlocks()).hasSize(1);
StmtsBlock block1 = doc1.getBlocks().get(0);
OpsBlock block1 = doc1.getBlocks().get(0);
assertThat(block1.getBindings()).hasSize(0);
assertThat(block1.getTags()).hasSize(0);
assertThat(block1.getOps()).hasSize(3);

129
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/OpVariationTests.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class OpVariationTests {
private final static Logger logger = LogManager.getLogger(OpVariationTests.class);
@Test
public void testListOpsOnly() {
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"ops:\n" +
" - first op\n" +
" - second op\n"
);
assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawOpDefs()).hasSize(2);
List<RawOpDef> ops = doc.getRawOpDefs();
RawOpDef s0 = ops.get(0);
assertThat(s0.getName()).isEqualTo("stmt1");
assertThat(s0.getStmt()).isEqualTo("first op");
RawOpDef s1 = ops.get(1);
assertThat(s1.getName()).isEqualTo("stmt2");
assertThat(s1.getStmt()).isEqualTo("second op");
}
@Test
public void testSingleEntryMapStmtsOnly() {
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"ops:\n" +
" - s1: op one\n" +
" - s2: op two\n"
);
assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawOpDefs()).hasSize(2);
List<RawOpDef> ops = doc.getRawOpDefs();
assertThat(ops.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(ops.get(0).getName()).isEqualTo("s1");
assertThat(ops.get(0).getStmt()).isEqualTo("op one");
assertThat(ops.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(ops.get(1).getName()).isEqualTo("s2");
assertThat(ops.get(1).getStmt()).isEqualTo("op two");
}
@Test
public void testMapStmtsOnly() {
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"ops:\n" +
" - name: s1\n" +
" stmt: op one\n" +
" - name: s2\n" +
" stmt: op two\n"
);
assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawOpDefs()).hasSize(2);
List<RawOpDef> stmts = doc.getRawOpDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("op one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("op two");
}
/**
* This test uses the compatible names for the sake of demonstration. Users should know that they can use op and
* statement terms interchangeably in op templates.
*/
@Test
public void testMixedForms() {
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList docs = ysl.loadString(
"statement:\n" +
" - name: s1\n" +
" stmt: statement one\n" +
" - statement two\n" +
" - s3: statement three\n" +
" - ST4: statement four\n" +
" type: organic\n"
);
assertThat(docs.getOpsDocs()).hasSize(1);
RawOpsDoc doc = docs.getOpsDocs().get(0);
assertThat(doc.getRawOpDefs()).hasSize(4);
List<RawOpDef> stmts = doc.getRawOpDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("stmt2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
assertThat(stmts.get(2)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(2).getName()).isEqualTo("s3");
assertThat(stmts.get(2).getStmt()).isEqualTo("statement three");
assertThat(stmts.get(3)).isOfAnyClassIn(RawOpDef.class);
assertThat(stmts.get(3).getName()).isEqualTo("ST4");
assertThat(stmts.get(3).getStmt()).isEqualTo("statement four");
assertThat(stmts.get(3).getParams().get("type")).isEqualTo("organic");
}
}

60
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoaderTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlTemplateLoaderTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -25,16 +25,16 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class RawYamlStatementLoaderTest {
private final static Logger logger = LogManager.getLogger(RawYamlStatementLoaderTest.class);
public class RawYamlTemplateLoaderTest {
private final static Logger logger = LogManager.getLogger(RawYamlTemplateLoaderTest.class);
@Test
public void testLoadPropertiesBlock() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList rawBlockDocs = ysl.loadPath(logger, "testdocs/rawblock.yaml");
assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1);
RawStmtsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0);
assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1);
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml");
assertThat(rawBlockDocs.getOpsDocs()).hasSize(1);
RawOpsDoc rawBlockDoc = rawBlockDocs.getOpsDocs().get(0);
assertThat(rawBlockDoc.getRawOpDefs()).hasSize(1);
assertThat(rawBlockDoc.getBindings()).hasSize(1);
assertThat(rawBlockDoc.getName()).isEqualTo("name");
assertThat(rawBlockDoc.getTags()).hasSize(1);
@ -43,30 +43,30 @@ public class RawYamlStatementLoaderTest {
@Test
public void testLoadFullFormat() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
List<RawStmtsDoc> rawStmtsDocs = erthing.getStmtsDocs();
assertThat(rawStmtsDocs).hasSize(2);
RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0);
List<RawStmtsBlock> blocks = rawStmtsDoc.getBlocks();
assertThat(rawStmtsDoc.getName()).isEqualTo("doc1");
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
assertThat(rawOpsDocs).hasSize(2);
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
assertThat(rawOpsDoc.getName()).isEqualTo("doc1");
assertThat(blocks).hasSize(1);
RawStmtsBlock rawStmtsBlock = blocks.get(0);
assertThat(rawStmtsBlock.getName()).isEqualTo("block0");
RawOpsBlock rawOpsBlock = blocks.get(0);
assertThat(rawOpsBlock.getName()).isEqualTo("block0");
}
@Test
public void testLoadScenarios() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
List<RawStmtsDoc> rawStmtsDocs = erthing.getStmtsDocs();
assertThat(rawStmtsDocs).hasSize(2);
RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0);
List<RawStmtsBlock> blocks = rawStmtsDoc.getBlocks();
assertThat(rawStmtsDoc.getDesc()).isEqualTo(
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
assertThat(rawOpsDocs).hasSize(2);
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
assertThat(rawOpsDoc.getDesc()).isEqualTo(
"a quintessential description - this is superseded by dedicated specification tests and will be removed");
RawScenarios rawScenarios = rawStmtsDoc.getRawScenarios();
RawScenarios rawScenarios = rawOpsDoc.getRawScenarios();
assertThat(rawScenarios.getScenarioNames()).containsExactly("default", "schema-only");
Map<String, String> defaultScenario = rawScenarios.getNamedScenario("default");
assertThat(defaultScenario.keySet())
@ -79,16 +79,16 @@ public class RawYamlStatementLoaderTest {
assertThat(schemaOnlyScenario.values())
.containsExactly("run driver=blah tags=phase:schema");
assertThat(rawStmtsDoc.getName()).isEqualTo("doc1");
assertThat(rawOpsDoc.getName()).isEqualTo("doc1");
assertThat(blocks).hasSize(1);
RawStmtsBlock rawStmtsBlock = blocks.get(0);
assertThat(rawStmtsBlock.getName()).isEqualTo("block0");
RawOpsBlock rawOpsBlock = blocks.get(0);
assertThat(rawOpsBlock.getName()).isEqualTo("block0");
}
@Test
public void testErrorMsg() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/badyamlfile.yaml");
RawOpsLoader ysl = new RawOpsLoader();
RawOpsDocList erthing = ysl.loadPath("testdocs/badyamlfile.yaml");
}
}

125
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/StmtVariationTests.java
View File

@ -1,125 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class StmtVariationTests {
private final static Logger logger = LogManager.getLogger(StmtVariationTests.class);
@Test
public void testListStmtsOnly() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statements:\n" +
" - first statement\n" +
" - second statement\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
RawStmtDef s0 = stmts.get(0);
assertThat(s0.getName()).isEqualTo("stmt1");
assertThat(s0.getStmt()).isEqualTo("first statement");
RawStmtDef s1 = stmts.get(1);
assertThat(s1.getName()).isEqualTo("stmt2");
assertThat(s1.getStmt()).isEqualTo("second statement");
}
@Test
public void testSingleEntryMapStmtsOnly() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statements:\n" +
" - s1: statement one\n" +
" - s2: statement two\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
}
@Test
public void testMapStmtsOnly() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statements:\n" +
" - name: s1\n" +
" stmt: statement one\n" +
" - name: s2\n" +
" stmt: statement two\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(2);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("s2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
}
@Test
public void testMixedForms() {
RawStmtsLoader ysl = new RawStmtsLoader();
RawStmtsDocList docs = ysl.loadString(logger,
"statement:\n" +
" - name: s1\n" +
" stmt: statement one\n" +
" - statement two\n" +
" - s3: statement three\n" +
" - ST4: statement four\n" +
" type: organic\n"
);
assertThat(docs.getStmtsDocs()).hasSize(1);
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
assertThat(doc.getRawStmtDefs()).hasSize(4);
List<RawStmtDef> stmts = doc.getRawStmtDefs();
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(0).getName()).isEqualTo("s1");
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(1).getName()).isEqualTo("stmt2");
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
assertThat(stmts.get(2)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(2).getName()).isEqualTo("s3");
assertThat(stmts.get(2).getStmt()).isEqualTo("statement three");
assertThat(stmts.get(3)).isOfAnyClassIn(RawStmtDef.class);
assertThat(stmts.get(3).getName()).isEqualTo("ST4");
assertThat(stmts.get(3).getStmt()).isEqualTo("statement four");
assertThat(stmts.get(3).getParams().get("type")).isEqualTo("organic");
}
}

18
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java
View File

@ -18,10 +18,11 @@ package io.nosqlbench.engine.api.activityconfig.specifications;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.nb.spectest.api.STAssemblyValidator;
import io.nosqlbench.nb.spectest.core.STNodeAssembly;
import io.nosqlbench.nb.spectest.loaders.STDefaultLoader;
@ -107,8 +108,8 @@ public class YamlSpecValidator implements STAssemblyValidator {
}.getType();
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml, Map.of());
List<OpTemplate> stmts = stmtsDocs.getStmts();
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
List<OpTemplate> stmts = stmtsDocs.getOps();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());
try {
@ -142,8 +143,6 @@ public class YamlSpecValidator implements STAssemblyValidator {
private void validateYamlWithJson(String desc, String yaml, String json, STNodeReference testset, boolean debug) {
System.out.format("%-40s", "- checking yaml->json");
// StmtsDocList stmts = StatementsLoader.loadString(yaml);
try {
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml);
JsonElement elem = null;
@ -174,10 +173,9 @@ public class YamlSpecValidator implements STAssemblyValidator {
throw new RuntimeException("unknown type in comparator: " + json);
}
} catch (Exception e) {
StringBuilder sb = new StringBuilder();
sb.append("error while verifying model:\n");
sb.append(" path: ").append(testset.getPath().toString()).append("\n");
sb.append(" line: ").append(testset.getLineNumber()).append("\n");
String sb = "error while verifying model:\n" +
" path: " + testset.getPath().toString() + "\n" +
" line: " + testset.getLineNumber() + "\n";
logger.error(sb + ": " + e.getMessage(), e);
throw new RuntimeException(e);

28
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtDetailOverrideTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpDetailOverrideTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -26,21 +26,21 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StmtDetailOverrideTest {
private static final Logger logger = LogManager.getLogger(StmtDetailOverrideTest.class);
public class OpDetailOverrideTest {
private static final Logger logger = LogManager.getLogger(OpDetailOverrideTest.class);
@Test
public void testStmtOverrides() {
StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml");
OpsDocList doclist = OpsLoader.loadPath("testdocs/op_details.yaml", Map.of());
assertThat(doclist).isNotNull();
assertThat(doclist.getStmtDocs()).hasSize(1);
StmtsDoc doc1 = doclist.getStmtDocs().get(0);
OpsDoc doc1 = doclist.getStmtDocs().get(0);
assertThat(doc1.getBlocks()).hasSize(2);
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
OpsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1block0.getOps().size()).isEqualTo(1);
OpTemplate s = doc1block0.getOps().get(0);
assertThat(s.getName()).isEqualTo("block0--stmt1");
@ -49,25 +49,25 @@ public class StmtDetailOverrideTest {
assertThat(s.getParams()).hasSize(1);
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1"));
StmtsBlock doc1block1 = doc1.getBlocks().get(1);
List<OpTemplate> stmts = doc1block1.getOps();
assertThat(stmts).hasSize(4);
OpsBlock doc1block1 = doc1.getBlocks().get(1);
List<OpTemplate> ops = doc1block1.getOps();
assertThat(ops).hasSize(4);
s = stmts.get(0);
s = ops.get(0);
assertThat(s.getName()).isEqualTo("testblock1--stmt1");
assertThat(s.getStmt()).contains("astatement1");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--stmt1"));
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
s = stmts.get(1);
s = ops.get(1);
assertThat(s.getName()).isEqualTo("testblock1--s2name");
assertThat(s.getStmt()).contains("s2statement data");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s2name"));
assertThat(s.getBindings()).hasSize(1);
assertThat(s.getParams()).hasSize(1);
s = stmts.get(2);
s = ops.get(2);
assertThat(s.getName()).isEqualTo("testblock1--s3");
assertThat(s.getStmt()).contains("statement three");
assertThat(s.getTags()).containsEntry("tname1", "tval1");
@ -75,7 +75,7 @@ public class StmtDetailOverrideTest {
assertThat(s.getBindings()).hasSize(3);
assertThat(s.getParams()).hasSize(2);
s = stmts.get(3);
s = ops.get(3);
assertThat(s.getName()).isEqualTo("testblock1--s4");
assertThat(s.getStmt()).contains("statement 4");
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s4"));

48
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocListTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/OpsDocListTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import org.assertj.core.data.MapEntry;
import org.junit.jupiter.api.BeforeAll;
import org.apache.logging.log4j.Logger;
@ -29,10 +29,10 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class StmtsDocListTest {
public class OpsDocListTest {
private static final Logger logger = LogManager.getLogger(StmtsDocListTest.class);
private static StmtsDocList doclist;
private static final Logger logger = LogManager.getLogger(OpsDocListTest.class);
private static OpsDocList doclist;
private final LinkedHashMap<String, String> doc0bindings = new LinkedHashMap<String, String>() {{
put("b2", "b2d");
@ -50,17 +50,17 @@ public class StmtsDocListTest {
@BeforeAll
public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
doclist = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
}
@Test
public void testBlocksInheritDocData() {
assertThat(doclist).isNotNull();
assertThat(doclist.getStmtDocs()).hasSize(2);
StmtsDoc doc1 = doclist.getStmtDocs().get(0);
OpsDoc doc1 = doclist.getStmtDocs().get(0);
assertThat(doc1.getBlocks()).hasSize(1);
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
OpsBlock doc1block0 = doc1.getBlocks().get(0);
assertThat(doc1.getTags()).isEqualTo(doc0tags);
assertThat(doc1.getBindings()).isEqualTo(doc0bindings);
@ -75,29 +75,29 @@ public class StmtsDocListTest {
@Test
public void testStmtInheritsBlockData() {
StmtsDoc doc0 = doclist.getStmtDocs().get(0);
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getOps();
assertThat(stmts1).hasSize(2);
OpsDoc doc0 = doclist.getStmtDocs().get(0);
List<OpTemplate> ops1 = doc0.getBlocks().get(0).getOps();
assertThat(ops1).hasSize(2);
StmtsBlock block0 = doc0.getBlocks().get(0);
OpsBlock block0 = doc0.getBlocks().get(0);
assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue"));
assertThat(stmts1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(stmts1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(stmts1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
assertThat(ops1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(ops1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(ops1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
assertThat(stmts1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(stmts1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(stmts1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
assertThat(ops1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
assertThat(ops1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
assertThat(ops1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
}
@Test
public void testBlockLayersDocData() {
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
StmtsBlock block0 = doc1.getBlocks().get(0);
OpsDoc doc1 = doclist.getStmtDocs().get(1);
OpsBlock block0 = doc1.getBlocks().get(0);
Map<String, String> doc1block0tags = block0.getTags();
Map<String, String> doc1block0params = block0.getParamsAsText();
@ -119,16 +119,16 @@ public class StmtsDocListTest {
@Test
public void testStmtsGetter() {
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
List<OpTemplate> stmts = doc1.getStmts();
OpsDoc doc1 = doclist.getStmtDocs().get(1);
List<OpTemplate> stmts = doc1.getOpTemplates();
assertThat(stmts).hasSize(4);
}
@Test
public void testFilteredStmts() {
List<OpTemplate> stmts = doclist.getStmts("");
List<OpTemplate> stmts = doclist.getOps("");
assertThat(stmts).hasSize(6);
stmts = doclist.getStmts("root1:value23");
stmts = doclist.getOps("root1:value23");
assertThat(stmts).hasSize(2);
}

20
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedStmtOpTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,33 +16,35 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedStmtOpTest {
private static final Logger logger = LogManager.getLogger(ParsedStmtOpTest.class);
private static StmtsDocList doclist;
public class ParsedOpTemplateTest {
private static final Logger logger = LogManager.getLogger(ParsedOpTemplateTest.class);
private static OpsDocList doclist;
@BeforeAll
public static void testLoadYaml() {
doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml");
doclist = OpsLoader.loadPath("testdocs/bindings.yaml", Map.of());
}
@Test
public void testBasicParser() {
StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
OpsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
OpTemplate stmtDef0 = block0.getOps().get(0);
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
assertThat(parsed0.getMissing()).containsExactly("delta");
assertThat(parsed0.hasError()).isTrue();
StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
OpsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
OpTemplate stmtDef1 = block1.getOps().get(0);
ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
assertThat(parsed1.getMissing()).containsExactly();
@ -51,7 +53,7 @@ public class ParsedStmtOpTest {
@Test
public void testMultipleBindingUsage() {
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
OpsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
OpTemplate stmtDef0 = block2.getOps().get(0);
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();

13
adapters-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedOpTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,10 +16,11 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.config.standard.Param;
@ -68,9 +69,9 @@ public class ParsedOpTest {
params:
ps1: "param-one"
""";
StmtsDocList stmtsDocs = StatementsLoader.loadString(opt, cfg.getMap());
assertThat(stmtsDocs.getStmts().size()).isEqualTo(1);
OpTemplate opTemplate = stmtsDocs.getStmts().get(0);
OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
assertThat(stmtsDocs.getOps().size()).isEqualTo(1);
OpTemplate opTemplate = stmtsDocs.getOps().get(0);
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg);
assertThat(parsedOp.getAsFunctionOr("d1","invalid").apply(1L)).isEqualTo("one");

0
adapters-api/src/test/resources/testdocs/docs_blocks_stmts.yaml → adapters-api/src/test/resources/testdocs/docs_blocks_ops.yaml
View File

0
adapters-api/src/test/resources/testdocs/escaped_stmts.yaml → adapters-api/src/test/resources/testdocs/escaped_ops.yaml
View File

0
adapters-api/src/test/resources/testdocs/stmt_details.yaml → adapters-api/src/test/resources/testdocs/op_details.yaml
View File

0
adapters-api/src/test/resources/testdocs/statement_variants.yaml → adapters-api/src/test/resources/testdocs/op_variants.yaml
View File

49
docsys/pom.xml
View File

@ -94,7 +94,7 @@
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>3.1.0</version>
<version>3.1.1</version>
</dependency>
<dependency>
@ -133,51 +133,4 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

47
driver-cockroachdb/pom.xml
View File

@ -47,51 +47,4 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

50
driver-jdbc/pom.xml
View File

@ -41,54 +41,4 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

47
driver-jmx/pom.xml
View File

@ -42,52 +42,5 @@
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

48
driver-kafka/pom.xml
View File

@ -65,54 +65,6 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>confluent</id>

47
driver-pulsar/pom.xml
View File

@ -90,52 +90,5 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

45
engine-api/pom.xml
View File

@ -136,51 +136,6 @@
<filtering>false</filtering> <!-- exclusion from defaults -->
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

36
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -37,10 +37,10 @@ import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.activityimpl.uniform.DryRunOpDispenserWrapper;
@ -109,7 +109,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
}
@Override
public void initActivity() {
public synchronized void initActivity() {
initOrUpdateRateLimiters(this.activityDef);
}
@ -353,7 +353,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
*
* @param seq - The {@link OpSequence} to derive the defaults from
*/
public void setDefaultsFromOpSequence(OpSequence<?> seq) {
public synchronized void setDefaultsFromOpSequence(OpSequence<?> seq) {
Optional<String> strideOpt = getParams().getOptionalString("stride");
if (strideOpt.isEmpty()) {
String stride = String.valueOf(seq.getSequence().length);
@ -549,21 +549,21 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
StmtsDocList stmtsDocList = loadStmtsDocList();
OpsDocList opsDocList = loadStmtsDocList();
List<OpTemplate> unfilteredOps = stmtsDocList.getStmts();
List<OpTemplate> filteredOps = stmtsDocList.getStmts(tagfilter);
List<OpTemplate> unfilteredOps = opsDocList.getOps();
List<OpTemplate> filteredOps = opsDocList.getOps(tagfilter);
if (filteredOps.size() == 0) {
if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out
throw new BasicError("There were no active statements with tag filter '"
throw new BasicError("There were no active op templates with tag filter '"
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
} else {
// There were no ops, and it *wasn't* because they were all filtered out.
// In this case, let's try to synthesize the ops as long as at least a default driver was provided
if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
filteredOps = sotp.getSyntheticOpTemplates(stmtsDocList, getActivityDef().getParams());
filteredOps = sotp.getSyntheticOpTemplates(opsDocList, getActivityDef().getParams());
Objects.requireNonNull(filteredOps);
if (filteredOps.size() == 0) {
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + "'" +
@ -573,12 +573,12 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
throw new BasicError("""
No op templates were provided. You must provide one of these activity parameters:
1) workload=some.yaml
2) op='inline template
2) op='inline template'
3) driver=stdout (or any other drive that can synthesize ops)""");
}
}
if (filteredOps.size() == 0) {
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
throw new BasicError("There were no active op templates with tag filter '" + tagfilter + "'");
}
}
@ -613,8 +613,8 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
* taken as the only provided statement.</LI>
* <LI>If a 'yaml, or 'workload' parameter is provided, then the statements in that file
* are taken with their ratios </LI>
* <LI>Any provided tags filter is used to select only the statements which have matching
* tags. If no tags are provided, then all the found statements are included.</LI>
* <LI>Any provided tags filter is used to select only the op templates which have matching
* tags. If no tags are provided, then all the found op templates are included.</LI>
* <LI>The ratios and the 'seq' parameter are used to build a sequence of the ready operations,
* where the sequence length is the sum of the ratios.</LI>
* </OL>
@ -661,20 +661,20 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
return planner.resolve();
}
protected StmtsDocList loadStmtsDocList() {
protected OpsDocList loadStmtsDocList() {
try {
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (stmt.isPresent()) {
workloadSource = "commandline:" + stmt.get();
return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null);
} else if (op_yaml_loc.isPresent()) {
workloadSource = "yaml:" + op_yaml_loc.get();
return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");
}
return StmtsDocList.none();
return OpsDocList.none();
} catch (Exception e) {
throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);

12
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,6 +33,7 @@ import io.nosqlbench.engine.api.activityimpl.MotorState;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import static io.nosqlbench.engine.api.activityapi.core.RunState.*;
@ -460,18 +461,17 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
@Override
public synchronized void requestStop() {
if (motorState.get() == Running) {
RunState currentState = motorState.get();
if (Objects.requireNonNull(currentState) == Running) {
if (input instanceof Stoppable) {
((Stoppable) input).requestStop();
}
if (action instanceof Stoppable) {
((Stoppable) action).requestStop();
}
motorState.enterState(RunState.Stopping);
motorState.enterState(Stopping);
} else {
if (motorState.get() != Stopped && motorState.get() != Stopping) {
logger.warn(()->"attempted to stop motor " + this.getSlotId() + ": from non Running state:" + motorState.get());
}
logger.warn(() -> "attempted to stop motor " + this.getSlotId() + ": from non Running state:" + currentState);
}
}

11
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -20,9 +20,9 @@ import io.nosqlbench.api.config.standard.*;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
import io.nosqlbench.engine.api.activityimpl.OpMapper;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
@ -54,11 +54,12 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
public StandardActivity(ActivityDef activityDef) {
super(activityDef);
OpsDocList workload;
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (yaml_loc.isPresent()) {
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities");
workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities");
yamlmodel = workload.getConfigModel();
} else {
yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly();
@ -163,11 +164,11 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
@Override
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
List<OpTemplate> opTemplates = new ArrayList<>();
for (DriverAdapter adapter : adapters.values()) {
if (adapter instanceof SyntheticOpTemplateProvider sotp) {
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(stmtsDocList, cfg);
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(opsDocList, cfg);
opTemplates.addAll(newTemplates);
}
}

8
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivityType.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,8 +18,8 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.api.config.standard.NBConfigModel;
@ -69,7 +69,7 @@ public class StandardActivityType<A extends StandardActivity<?,?>> extends Simpl
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
if (op_yaml_loc.isPresent()) {
Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams());
StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities");
OpsDocList workload = OpsLoader.loadPath(op_yaml_loc.get(), disposable, "activities");
cfgModel=cfgModel.add(workload.getConfigModel());
}
NBConfiguration cfg = cfgModel.apply(activityDef.getParams());

22
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -16,10 +16,10 @@
package io.nosqlbench.engine.api.scenarios;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
@ -53,7 +53,7 @@ public class NBCLIScenarioParser {
.prefix("activities")
.prefix(includes)
.name(workload)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.first();
return found.isPresent();
}
@ -67,7 +67,7 @@ public class NBCLIScenarioParser {
.prefix("activities")
.prefix(includes)
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.first();
//
Content<?> workloadContent = found.orElseThrow();
@ -111,10 +111,10 @@ public class NBCLIScenarioParser {
.prefix(SEARCH_IN)
.prefix(includes)
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
OpsDocList scenariosYaml = OpsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
Scenarios scenarios = scenariosYaml.getDocScenarios();
String[] nameparts = scenarioName.split("\\.",2);
@ -314,12 +314,12 @@ public class NBCLIScenarioParser {
}
Content<?> content = NBIO.all().prefix(SEARCH_IN)
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
.name(referenced).extension(RawOpsLoader.YAML_EXTENSIONS)
.one();
StmtsDocList stmts = null;
OpsDocList stmts = null;
try {
stmts = StatementsLoader.loadContent(logger, content, Map.of());
stmts = OpsLoader.loadContent(content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;
@ -379,7 +379,7 @@ public class NBCLIScenarioParser {
List<Content<?>> activities = searchin
.prefix(includes)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.list();
return filterForScenarios(activities);

18
engine-api/src/test/java/io/nosqlbench/engine/api/scripting/GraalJsEvaluatorTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -24,6 +24,22 @@ import static org.assertj.core.api.Assertions.assertThat;
public class GraalJsEvaluatorTest {
// TODO:
/**
* This evaluator may benefit from being compiled. Presently it give a warning:
* <pre>{@code
* [To redirect Truffle log output to a file use one of the following options:
* * '--log.file=<path>' if the option is passed using a guest language launcher.
* * '-Dpolyglot.log.file=<path>' if the option is passed using the host Java launcher.
* * Configure logging using the polyglot embedding API.]
* [engine] WARNING: The polyglot context is using an implementation that does not support runtime compilation.
* The guest application code will therefore be executed in interpreted mode only.
* Execution only in interpreted mode will strongly impact the guest application performance.
* For more information on using GraalVM see https://www.graalvm.org/java/quickstart/.
* To disable this warning the '--engine.WarnInterpreterOnly=false' option or use the '-Dpolyglot.engine.WarnInterpreterOnly=false' system property.
* }</pre>
*/
@Test
public void testBasicOperations() {
GraalJsEvaluator<Long> ne = new GraalJsEvaluator<>(Long.class);

23
engine-api/src/test/java/io/nosqlbench/engine/api/templating/CommandTemplateTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,9 +18,10 @@ package io.nosqlbench.engine.api.templating;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -34,11 +35,11 @@ public class CommandTemplateTest {
@Test
public void testCommandTemplate() {
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
"statements:\n" +
OpsDocList opsDocs = OpsLoader.loadString("" +
"ops:\n" +
" - s1: test1=foo test2=bar",
Map.of());
OpTemplate optpl = stmtsDocs.getStmts().get(0);
OpTemplateFormat.yaml, Map.of(), null);
OpTemplate optpl = opsDocs.getOps().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
assertThat(ct.isStatic()).isTrue();
}
@ -46,14 +47,14 @@ public class CommandTemplateTest {
@Test
public void testCommandTemplateFormat() {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
"statements:\n" +
OpsDocList stmtsDocs = OpsLoader.loadString("" +
"ops:\n" +
" - s1: test1=foo test2={bar}\n" +
" bindings:\n" +
" bar: NumberNameToString();\n",
Map.of()
OpTemplateFormat.yaml, Map.of(), null
);
OpTemplate optpl = stmtsDocs.getStmts().get(0);
OpTemplate optpl = stmtsDocs.getOps().get(0);
CommandTemplate ct = new CommandTemplate(optpl);
String format = gson.toJson(ct);
logger.debug(format);

45
engine-cli/pom.xml
View File

@ -51,51 +51,6 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

12
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDu
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
import io.nosqlbench.engine.api.activityapi.input.InputType;
import io.nosqlbench.engine.api.activityapi.output.OutputType;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
import io.nosqlbench.engine.core.annotation.Annotators;
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
@ -165,7 +165,11 @@ public class NBCLI implements Function<String[], Integer> {
logger = LogManager.getLogger("NBCLI");
loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
logger.info(() -> "Configured scenario log at " + loggerConfig.getLogfileLocation());
if (logger.isInfoEnabled()) {
logger.info(() -> "Configured scenario log at " + loggerConfig.getLogfileLocation());
} else {
System.err.println("Configured scenario log at " + loggerConfig.getLogfileLocation());
}
logger.debug("Scenario log started");
// Global only processing
@ -304,7 +308,7 @@ public class NBCLI implements Function<String[], Integer> {
Optional<Content<?>> tocopy = NBIO.classpath()
.prefix("activities")
.prefix(options.wantsIncludes())
.name(resourceToCopy).extension(RawStmtsLoader.YAML_EXTENSIONS).first();
.name(resourceToCopy).extension(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocopy.isEmpty()) {

18
engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTemplateVarTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,9 +16,9 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.junit.jupiter.api.Test;
import java.util.List;
@ -33,13 +33,13 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getOps().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getStmts().get(0);
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
OpTemplate optpl2 = workload2.getOps().get(0);
System.out.println("op from cmd2:"+optpl2);
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
}
@ -50,8 +50,8 @@ public class NBCLIScenarioParserTemplateVarTest {
List<Cmd> cmds = opts.getCommands();
cmds.forEach(System.out::println);
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getStmts().get(0);
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
OpTemplate optpl1 = workload1.getOps().get(0);
System.out.println("op from cmd1:"+optpl1);
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
}

45
engine-clients/pom.xml
View File

@ -58,51 +58,6 @@
</includes>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

44
engine-core/pom.xml
View File

@ -93,49 +93,5 @@
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

12
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java
View File

@ -299,7 +299,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
break;
case Finished:
case Stopping:
throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
case Errored:
break;
// throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
default:
throw new RuntimeException("Unmatched run state:" + activity.getRunState());
@ -314,16 +316,16 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
case Running:
tally.awaitNoneOther(RunState.Running, RunState.Finished);
break;
case Errored:
case Stopping:
case Stopped:
tally.awaitNoneOther(RunState.Stopped, RunState.Finished);
tally.awaitNoneOther(RunState.Stopped, RunState.Finished, RunState.Errored);
break;
case Uninitialized:
break;
case Finished:
tally.awaitNoneOther(RunState.Finished);
break;
case Stopping:
throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
default:
throw new RuntimeException("Unmatched run state:" + activity.getRunState());
}
@ -417,7 +419,7 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
}
}
public void startActivity() {
public synchronized void startActivity() {
// we need an executor service to run motor threads on
startMotorExecutorService();
startRunningActivityThreads();

8
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/Scenario.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -277,8 +277,10 @@ public class Scenario implements Callable<ExecutionMetricsResult> {
}
Runtime.getRuntime().removeShutdownHook(scenarioShutdownHook);
scenarioShutdownHook.run();
var runHook = scenarioShutdownHook;
scenarioShutdownHook = null;
runHook.run();
logger.debug("removing scenario shutdown hook");
}
public void notifyException(Thread t, Throwable e) {
@ -340,7 +342,7 @@ public class Scenario implements Callable<ExecutionMetricsResult> {
}
}
public synchronized void finish() {
public void finish() {
logger.debug("finishing scenario");
endedAtMillis = System.currentTimeMillis(); //TODO: Make only one endedAtMillis assignment
if (this.state == State.Running) {

49
engine-docker/pom.xml
View File

@ -76,53 +76,4 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

43
engine-docs/pom.xml
View File

@ -66,49 +66,6 @@
</activation>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>

66
engine-extensions/pom.xml
View File

@ -45,7 +45,6 @@
<build>
<resources>
<resource>
<directory>src/main/java</directory>
@ -54,71 +53,6 @@
</includes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-javadoc-plugin</artifactId>-->
<!-- <version>3.1.1</version>-->
<!-- <configuration>-->
<!-- <doctitle>nosqlbench Extensions</doctitle>-->
<!-- <windowtitle>nosqlbench Extensions</windowtitle>-->
<!-- </configuration>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <id>attach-javadoc</id>-->
<!-- <goals>-->
<!-- <goal>jar</goal>-->
<!-- </goals>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
</plugins>
</build>
</project>

48
engine-rest/pom.xml
View File

@ -56,52 +56,4 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

4
engine-rest/src/main/java/io/nosqlbench/engine/rest/services/WorkSpace.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.rest.services;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView;

101
mvn-defaults/pom.xml
View File

@ -14,7 +14,9 @@
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.nosqlbench</groupId>
@ -34,8 +36,9 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<javadoc.name>nosqlbench</javadoc.name>
<jacoco.version>0.8.8</jacoco.version>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
@ -142,6 +145,32 @@
<version>1.10.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-core</artifactId>
<version>4.0-beta1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-legacy</artifactId>
<version>4.0-beta1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-legacy-core</artifactId>
<version>4.0-beta1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math4-legacy-exception</artifactId>
<version>4.0-beta1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-statistics-distribution</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
@ -270,7 +299,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.382</version>
<version>1.12.393</version>
</dependency>
<dependency>
@ -303,12 +332,6 @@
<version>2.12.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
@ -472,6 +495,7 @@
<version>1.7.1</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -510,7 +534,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.9.0</version>
<version>3.10.1</version>
<configuration>
<debug>true</debug>
<target>17</target>
@ -528,8 +552,10 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M6</version>
<configuration>
<argLine>-ea ${argLine}</argLine>
<systemPropertyVariables>
<Log4jContextSelector>org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
<Log4jContextSelector>
org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
</Log4jContextSelector>
</systemPropertyVariables>
<parallel>methods</parallel>
@ -546,6 +572,57 @@
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<phase>process-test-classes</phase>
<goals>
<goal>prepare-agent</goal>
</goals>
<configuration>
<destFile>${project.build.directory}/jacoco-data.exec</destFile>
</configuration>
</execution>
<execution>
<id>report</id>
<phase>verify</phase>
<goals>
<goal>report</goal>
</goals>
<configuration>
<dataFile>${project.build.directory}/jacoco-data.exec</dataFile>
<outputDirectory>${project.build.directory}/jacoco-reports</outputDirectory>
</configuration>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
@ -706,7 +783,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<version>3.10.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>

48
nb-annotations/pom.xml
View File

@ -45,52 +45,4 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

46
nb-api/pom.xml
View File

@ -150,52 +150,6 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

Some files were not shown because too many files have changed in this diff Show More