mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
Merge branch 'main' into nosqlbench-948-testcontainers
This commit is contained in:
commit
cb073b0abd
24
.github/workflows/build.yml
vendored
24
.github/workflows/build.yml
vendored
@ -5,9 +5,9 @@ name: build
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "!RELEASE_NOTES.md"
|
||||
- "!PREVIEW_NOTES.md"
|
||||
tags:
|
||||
- "![0-9]+.[0-9]+.[0-9]+-preview"
|
||||
- "![0-9]+.[0-9]+.[0-9]+-release"
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
@ -42,15 +42,15 @@ jobs:
|
||||
- name: Running tests
|
||||
run: mvn -B test
|
||||
|
||||
- name: Collecting reports
|
||||
run: tar -cvf codecov-report.tar target/coverage-report/**/*
|
||||
# - name: Collecting reports
|
||||
# run: tar -cvf codecov-report.tar target/coverage-report/**/*
|
||||
|
||||
- name: Uploading test coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
name: codecov-report
|
||||
path: codecov-report.tar
|
||||
# - name: Uploading test coverage
|
||||
# uses: actions/upload-artifact@v3
|
||||
# with:
|
||||
# node-version: '16'
|
||||
# name: codecov-report
|
||||
# path: codecov-report.tar
|
||||
|
||||
- name: Collecting logfiles
|
||||
if: success() || failure()
|
||||
@ -76,7 +76,7 @@ jobs:
|
||||
name: exported-docs
|
||||
path: exported_docs.zip
|
||||
|
||||
docs:
|
||||
builddocs:
|
||||
needs: build
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
|
4
.github/workflows/codeql-analysis.yml
vendored
4
.github/workflows/codeql-analysis.yml
vendored
@ -14,6 +14,10 @@ name: "CodeQL"
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
tags:
|
||||
- "![0-9]+.[0-9]+.[0-9]+-preview"
|
||||
- "![0-9]+.[0-9]+.[0-9]+-release"
|
||||
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ main ]
|
||||
|
231
.github/workflows/preview.yml
vendored
231
.github/workflows/preview.yml
vendored
@ -1,9 +1,12 @@
|
||||
name: preview
|
||||
|
||||
# This workflow should run for any push which is a preview build.
|
||||
# Preview builds are indicated with changes to PREVIEW_NOTES.md.
|
||||
# Preview builds are indicated with a tag that ends in -preview
|
||||
# In order to effect this, you can do the following:
|
||||
# scripts/release-notes.sh > PREVIEW_NOTES.md
|
||||
# scripts/tag-preview-build
|
||||
# If you want to replace a build, you can use -f and then
|
||||
# push the update to the tag like push -f origin <tag>,
|
||||
# although this should never be done!
|
||||
|
||||
on:
|
||||
push:
|
||||
@ -13,17 +16,17 @@ on:
|
||||
jobs:
|
||||
preview-build:
|
||||
runs-on: ubuntu-20.04
|
||||
outputs:
|
||||
preview_version: ${{ steps.versions.outputs.PREVIEW_VERSION }}
|
||||
docker_tags: ${{ steps.versions.outputs.DOCKER_TAGS }}
|
||||
steps:
|
||||
|
||||
- name: checkout repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
|
||||
- name: setup java
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
distribution: 'temurin'
|
||||
java-version: '17'
|
||||
java-package: jdk
|
||||
@ -44,40 +47,38 @@ jobs:
|
||||
df -h
|
||||
|
||||
- name: Cache Maven packages
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v3.2.3
|
||||
with:
|
||||
node-version: '16'
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: ${{ runner.os }}-m2
|
||||
|
||||
- name: read versions
|
||||
id: versions
|
||||
run: |
|
||||
set -x
|
||||
PREVIEW_VERSION=$(scripts/get-preview-version.sh)
|
||||
echo "PREVIEW_VERSION=${PREVIEW_VERSION}" >> $GITHUB_ENV
|
||||
echo "PREVIEW_VERSION=${PREVIEW_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${PREVIEW_VERSION},nosqlbench/nosqlbench:preview" >> $GITHUB_ENV
|
||||
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${PREVIEW_VERSION},nosqlbench/nosqlbench:preview" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: build preview revision
|
||||
run: |
|
||||
mvn clean package -Drevision="${{ env.PREVIEW_VERSION }}"
|
||||
|
||||
- name: Setup docker buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
with:
|
||||
node-version: '16'
|
||||
uses: docker/setup-buildx-action@v2.2.1
|
||||
|
||||
- name: docker hub login
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@v2.1.0
|
||||
with:
|
||||
node-version: '16'
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: docker test build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@v3.3.0
|
||||
with:
|
||||
node-version: '16'
|
||||
context: .
|
||||
file: Dockerfile
|
||||
pull: true
|
||||
@ -99,7 +100,6 @@ jobs:
|
||||
- name: upload artifacts
|
||||
uses: actions/upload-artifact@v1
|
||||
with:
|
||||
node-version: '16'
|
||||
name: binaries
|
||||
path: staging
|
||||
|
||||
@ -117,18 +117,12 @@ jobs:
|
||||
id: prepare_summary
|
||||
run: |
|
||||
#summary=$(scripts/release-notes.sh)
|
||||
summary=$(cat RELEASENOTES.md)
|
||||
summary=$(cat PREVIEW_NOTES.md)
|
||||
summary="${summary//'%'/'%25'}"
|
||||
summary="${summary//$'\n'/'%0A'}"
|
||||
summary="${summary//$'\r'/'%0D'}"
|
||||
echo "release_summary=$summary" >> $GITHUB_STATE
|
||||
|
||||
- name: tag preview version
|
||||
run: |
|
||||
git tag nb5-${{ env.PREVIEW_VERSION }}
|
||||
git tag -f nb5-preview
|
||||
git push --tags
|
||||
|
||||
- name: bump minor version
|
||||
run: |
|
||||
scripts/bump-minor-version
|
||||
@ -136,7 +130,6 @@ jobs:
|
||||
- name: docker push to hub
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
node-version: '16'
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
file: Dockerfile
|
||||
@ -144,125 +137,107 @@ jobs:
|
||||
push: true
|
||||
tags: ${{ env.DOCKER_TAGS }}
|
||||
|
||||
# https://github.com/softprops/action-gh-release
|
||||
- name: create github release
|
||||
id: create_github_release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
uses: softprops/action-gh-release@v0.1.15
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
node-version: '16'
|
||||
tag_name: ${{ env.PREVIEW_VERSION }}
|
||||
release_name: Release ${{ env.PREVIEW_VERSION }}
|
||||
# body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
# body_path: PREVIEW_NOTES.md
|
||||
draft: false
|
||||
prerelease: true
|
||||
body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
generate_release_notes: true
|
||||
name: Preview ${{ env.PREVIEW_VERSION }}
|
||||
fail_on_unmatched_files: true
|
||||
tag_name: ${{ env.PREVIEW_VERSION }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
files: |
|
||||
nb5/target/nb5
|
||||
nb5/target/nb5.jar
|
||||
|
||||
- name: upload nb.jar to github release
|
||||
id: upload-nb-jar
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
node-version: '16'
|
||||
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
asset_path: nb5/target/nb5.jar
|
||||
asset_name: nb5.jar
|
||||
asset_content_type: application/octet-stream
|
||||
|
||||
- name: upload nb binary to github release
|
||||
id: upload-nb-binary
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
node-version: '16'
|
||||
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
asset_path: nb5/target/nb5
|
||||
asset_name: nb5
|
||||
asset_content_type: application/octet-stream
|
||||
# - name: create github release
|
||||
# id: create_github_release
|
||||
# uses: actions/create-release@v1
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# with:
|
||||
# tag_name: ${{ env.PREVIEW_VERSION }}
|
||||
# release_name: Release ${{ env.PREVIEW_VERSION }}
|
||||
# draft: false
|
||||
# prerelease: true
|
||||
# body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
#
|
||||
# - name: upload nb.jar to github release
|
||||
# id: upload-nb-jar
|
||||
# uses: actions/upload-release-asset@v1
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# with:
|
||||
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
# asset_path: nb5/target/nb5.jar
|
||||
# asset_name: nb5.jar
|
||||
# asset_content_type: application/octet-stream
|
||||
#
|
||||
# - name: upload nb binary to github release
|
||||
# id: upload-nb-binary
|
||||
# uses: actions/upload-release-asset@v1
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# with:
|
||||
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
# asset_path: nb5/target/nb5
|
||||
# asset_name: nb5
|
||||
# asset_content_type: application/octet-stream
|
||||
|
||||
- name: Archive Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
name: test-results
|
||||
path: |
|
||||
[a-zA-Z]**/logs/*
|
||||
|
||||
# javadocs:
|
||||
# needs: release
|
||||
# runs-on: ubuntu-20.04
|
||||
# steps:
|
||||
# - name: set git username
|
||||
# run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
|
||||
# - name: set git email
|
||||
# run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
|
||||
# - name: download javadocs
|
||||
# uses: actions/download-artifact@v3
|
||||
# with:
|
||||
# name: javadoc
|
||||
# - run: ls -la
|
||||
# - name: unpackage javadoc
|
||||
# run: unzip nosqlbench-*-javadoc.jar
|
||||
# - run: ls -la
|
||||
# - name: Push javadocs
|
||||
# uses: ad-m/github-push-action@master
|
||||
# with:
|
||||
# repository: nosqlbench/nosqlbench-javadoc
|
||||
# github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# branch: main
|
||||
#
|
||||
#
|
||||
# docs:
|
||||
# needs: release
|
||||
# runs-on: ubuntu-20.04
|
||||
# steps:
|
||||
#
|
||||
# - name: set git username
|
||||
# run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
|
||||
#
|
||||
# - name: set git email
|
||||
# run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
|
||||
#
|
||||
# - name: download guidebook
|
||||
# uses: actions/download-artifact@v1
|
||||
# with:
|
||||
# name: guidebook
|
||||
# path: guidebook
|
||||
#
|
||||
# - run: ls -la
|
||||
#
|
||||
# - name: clone nosqlbench-docs
|
||||
# env:
|
||||
# NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
|
||||
# NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
|
||||
# run: |
|
||||
# git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-docs.git nosqlbench-docs
|
||||
# cd nosqlbench-docs
|
||||
# echo "files listing"
|
||||
# find .
|
||||
# git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-docs.git
|
||||
# git remote -v
|
||||
#
|
||||
# Disabling this because it will be replaced soon.
|
||||
# - name: push changes
|
||||
# env:
|
||||
# NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
|
||||
# NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
|
||||
# run: |
|
||||
# rsync -av --delete guidebook/guidebook/ nosqlbench-docs/docs/
|
||||
# echo "docs.nosqlbench.io" > nosqlbench-docs/docs/CNAME
|
||||
# cd nosqlbench-docs
|
||||
# git add docs
|
||||
# git add -u
|
||||
# CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
|
||||
# echo "found $CHANGES to push for doc updates"
|
||||
# if (( $CHANGES > 0 ))
|
||||
# then
|
||||
# git commit -m"docs update for $GITHUB_REF"
|
||||
# git push
|
||||
# fi
|
||||
# This triggers a preview build by cascading the tag to the builddocs repo.
|
||||
# The builddocs repo then pushes to preview or release depending on the tag.
|
||||
|
||||
preview-docs:
|
||||
needs: preview-build
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: import env vars
|
||||
run: |
|
||||
echo "PREVIEW_VERSION=${{ needs.preview-build.outputs.preview_version }}" >> $GITHUB_ENV
|
||||
echo "DOCKER_TAGS=${{ needs.preview-build.outputs.docker_tags }}" >> $GITHUB_ENV
|
||||
- name: clone nosqlbench-build-docs
|
||||
run: |
|
||||
git clone https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git nosqlbench-build-docs
|
||||
cd nosqlbench-build-docs
|
||||
echo "files listing"
|
||||
find .
|
||||
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git
|
||||
git remote -v
|
||||
|
||||
- name: set CNAME
|
||||
run: |
|
||||
echo "builddocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME
|
||||
|
||||
- name: commit changes
|
||||
run: |
|
||||
cd nosqlbench-build-docs
|
||||
git add exported_docs.zip
|
||||
|
||||
- name: tag-preview-build
|
||||
run: |
|
||||
cd nosqlbench-build-docs
|
||||
git tag -f ${{ env.PREVIEW_VERSION }}
|
||||
|
||||
- name: push changes
|
||||
env:
|
||||
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
|
||||
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
|
||||
run: |
|
||||
set -x
|
||||
cd nosqlbench-build-docs
|
||||
CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
|
||||
git push -f --tags
|
||||
echo "push completed"
|
||||
|
152
.github/workflows/release.yml
vendored
152
.github/workflows/release.yml
vendored
@ -1,9 +1,12 @@
|
||||
name: release
|
||||
|
||||
# This workflow should run for any push which is a preview build.
|
||||
# Preview builds are indicated with changes to PREVIEW_NOTES.md.
|
||||
# This workflow should run for any push which is a release build.
|
||||
# Release builds are indicated with a tag that ends in -release
|
||||
# In order to effect this, you can do the following:
|
||||
# scripts/release-notes.sh > RELEASE_NOTES.md
|
||||
# scripts/tag-release-build
|
||||
# If you want to replace a build, you can use -f and then
|
||||
# push the update to the tag like push -f origin <tag>,
|
||||
# although this should never be done!
|
||||
|
||||
on:
|
||||
push:
|
||||
@ -17,13 +20,10 @@ jobs:
|
||||
|
||||
- name: checkout repo
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
|
||||
- name: setup java
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
distribution: 'temurin'
|
||||
java-version: '17'
|
||||
java-package: jdk
|
||||
@ -44,9 +44,8 @@ jobs:
|
||||
df -h
|
||||
|
||||
- name: Cache Maven packages
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v3.2.3
|
||||
with:
|
||||
node-version: '16'
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: ${{ runner.os }}-m2
|
||||
@ -54,30 +53,29 @@ jobs:
|
||||
- name: read versions
|
||||
run: |
|
||||
set -x
|
||||
RELEASE_REVISION=$(scripts/get-release-version.sh)
|
||||
echo "RELEASE_REVISION=${RELEASE_REVISION}" >> $GITHUB_ENV
|
||||
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${RELEASE_REVISION},nosqlbench/nosqlbench:latest" >> $GITHUB_ENV
|
||||
RELEASE_VERSION=$(scripts/get-release-version.sh)
|
||||
RELEASE_VERSION_NUMERIC=$(scripts/get-release-version.sh -n)
|
||||
RELEASE_VERSION_SHORT=$(scripts/get-release-version.sh -s)
|
||||
echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV
|
||||
echo "RELEASE_VERSION_NUMERIC=${RELEASE_VERSION_NUMERIC}" >> $GITHUB_ENV
|
||||
echo "DOCKER_TAGS=nosqlbench/nosqlbench:${RELEASE_VERSION_NUMERIC},nosqlbench/nosqlbench:${RELEASE_VERSION_SHORT}-latest" >> $GITHUB_ENV
|
||||
|
||||
- name: build release revision
|
||||
run: |
|
||||
mvn clean package -Drevision="${{ env.RELEASE_REVISION }}"
|
||||
mvn clean package -Drevision="${{ env.RELEASE_VERSION }}"
|
||||
|
||||
- name: Setup docker buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
with:
|
||||
node-version: '16'
|
||||
uses: docker/setup-buildx-action@v2.2.1
|
||||
|
||||
- name: docker hub login
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@v2.1.0
|
||||
with:
|
||||
node-version: '16'
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: docker test build
|
||||
uses: docker/build-push-action@v2
|
||||
uses: docker/build-push-action@v3.3.0
|
||||
with:
|
||||
node-version: '16'
|
||||
context: .
|
||||
file: Dockerfile
|
||||
pull: true
|
||||
@ -87,7 +85,7 @@ jobs:
|
||||
|
||||
- name: sanity check docker image
|
||||
run: |
|
||||
docker run --rm nosqlbench/nosqlbench:${{ env.RELEASE_REVISION }} --version
|
||||
docker run --rm nosqlbench/nosqlbench:${{ env.RELEASE_VERSION_NUMERIC }} --version
|
||||
|
||||
- name: bundle artifacts
|
||||
run: |
|
||||
@ -99,40 +97,26 @@ jobs:
|
||||
- name: upload artifacts
|
||||
uses: actions/upload-artifact@v1
|
||||
with:
|
||||
node-version: '16'
|
||||
name: binaries
|
||||
path: staging
|
||||
|
||||
# - name: generate javadoc
|
||||
# run: mvn javadoc:aggregate-jar
|
||||
# continue-on-error: false
|
||||
#
|
||||
# - name: upload javadoc
|
||||
# uses: actions/upload-artifact@v3
|
||||
# with:
|
||||
# name: javadoc
|
||||
# path: target/nosqlbench-*-javadoc.jar
|
||||
# - name: generate javadoc
|
||||
# run: mvn javadoc:aggregate-jar
|
||||
# continue-on-error: false
|
||||
#
|
||||
# - name: upload javadoc
|
||||
# uses: actions/upload-artifact@v3
|
||||
# with:
|
||||
# name: javadoc
|
||||
# path: target/nosqlbench-*-javadoc.jar
|
||||
|
||||
- name: prepare release summary
|
||||
id: prepare_summary
|
||||
- name: bump minor version
|
||||
run: |
|
||||
#summary=$(scripts/release-notes.sh)
|
||||
summary=$(cat RELEASENOTES.md)
|
||||
summary="${summary//'%'/'%25'}"
|
||||
summary="${summary//$'\n'/'%0A'}"
|
||||
summary="${summary//$'\r'/'%0D'}"
|
||||
echo "release_summary=$summary" >> $GITHUB_STATE
|
||||
|
||||
- name: tag release
|
||||
run: |
|
||||
git tag nb5-${{ env.RELEASE_REVISION }}
|
||||
git tag -f nb5-release
|
||||
git push --tags
|
||||
scripts/bump-minor-version
|
||||
|
||||
- name: docker push to hub
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
node-version: '16'
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
file: Dockerfile
|
||||
@ -140,48 +124,62 @@ jobs:
|
||||
push: true
|
||||
tags: ${{ env.DOCKER_TAGS }}
|
||||
|
||||
# https://github.com/softprops/action-gh-release
|
||||
- name: create github release
|
||||
id: create_github_release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
uses: softprops/action-gh-release@v0.1.15
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
with:
|
||||
node-version: '16'
|
||||
tag_name: ${{ env.RELEASE_REVISION }}
|
||||
release_name: Release ${{ env.RELEASE_REVISION }}
|
||||
# body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
# body_path: PREVIEW_NOTES.md
|
||||
draft: false
|
||||
prerelease: true
|
||||
body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
generate_release_notes: true
|
||||
name: Release ${{ env.RELEASE_VERSION }}
|
||||
fail_on_unmatched_files: true
|
||||
tag_name: ${{ env.RELEASE_VERSION }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
files: |
|
||||
nb5/target/nb5
|
||||
nb5/target/nb5.jar
|
||||
|
||||
- name: upload nb.jar to github release
|
||||
id: upload-nb-jar
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
node-version: '16'
|
||||
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
asset_path: nb5/target/nb5.jar
|
||||
asset_name: nb5.jar
|
||||
asset_content_type: application/octet-stream
|
||||
|
||||
- name: upload nb binary to github release
|
||||
id: upload-nb-binary
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
node-version: '16'
|
||||
upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
asset_path: nb5/target/nb5
|
||||
asset_name: nb5
|
||||
asset_content_type: application/octet-stream
|
||||
# - name: create github release
|
||||
# id: create_github_release
|
||||
# uses: actions/create-release@v1
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# with:
|
||||
# tag_name: ${{ env.RELEASE_VERSION }}
|
||||
# release_name: Release ${{ env.RELEASE_VERSION }}
|
||||
# draft: false
|
||||
# prerelease: true
|
||||
# body: ${{ steps.prepare_summary.outputs.release_summary }}
|
||||
#
|
||||
# - name: upload nb.jar to github release
|
||||
# id: upload-nb-jar
|
||||
# uses: actions/upload-release-asset@v1
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# with:
|
||||
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
# asset_path: nb5/target/nb5.jar
|
||||
# asset_name: nb5.jar
|
||||
# asset_content_type: application/octet-stream
|
||||
#
|
||||
# - name: upload nb binary to github release
|
||||
# id: upload-nb-binary
|
||||
# uses: actions/upload-release-asset@v1
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# with:
|
||||
# upload_url: ${{ steps.create_github_release.outputs.upload_url }}
|
||||
# asset_path: nb5/target/nb5
|
||||
# asset_name: nb5
|
||||
# asset_content_type: application/octet-stream
|
||||
|
||||
- name: Archive Test Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
node-version: '16'
|
||||
name: test-results
|
||||
path: |
|
||||
[a-zA-Z]**/logs/*
|
||||
|
@ -1,7 +1,33 @@
|
||||
- e4e94f9e8 (HEAD -> main) support version bumping
|
||||
- 2b85c1cce (HEAD -> main, tag: 5.17.1-preview) versions and release action update
|
||||
- 2d3263cfb make release tag distinct
|
||||
- 639dbf2d6 update action version
|
||||
- 1b2a773ce update files paths
|
||||
- 1986b8cdb replace release action
|
||||
- 7047f4486 (origin/main) remove second tagging
|
||||
- 910e871a8 remove extra node version parameters
|
||||
- 6b0633d4a (tag: nb5-preview, tag: nb5-5.17.1-preview) update notes filenames
|
||||
- be65f812e allow forcing preview tag update
|
||||
- 2a43dd077 Merge branch 'main' of github.com:nosqlbench/nosqlbench
|
||||
- 6a2132669 actions testing
|
||||
- 4bd0513f9 Merge pull request #950 from nosqlbench/nosqlbench-949-sigint
|
||||
- 086451794 preview build testing
|
||||
- 32486b3d6 unbreak the break signal
|
||||
- 6d07acb8c (origin/nosqlbench-949-sigint, nosqlbench-949-sigint) unbreak the break signal
|
||||
- c6aa66f50 Update DOWNLOADS.md
|
||||
- 5b90831a9 Update DOWNLOADS.md
|
||||
- 3e3427fca update job names
|
||||
- 3f0417c96 fix preview trigger
|
||||
- 29ed32d91 update revision
|
||||
- 24657248f use tags only for preview and release triggers
|
||||
- e254eafa8 check for main branch on preview and release
|
||||
- 0ae3bd085 make preview and release tag based
|
||||
- 027fc4a89 version, tagging updates
|
||||
- 402154e4c docs and scripts
|
||||
- ee2bda24e preview testing
|
||||
- e4e94f9e8 support version bumping
|
||||
- 8a2c696eb allow bumping versions
|
||||
- c19b2388e (origin/prerelease-testing, origin/main, prerelease-testing) standardize on main/preview/release naming
|
||||
- 5facaddfe (tag: 5.17.1-prerelease) actions testing
|
||||
- c19b2388e (origin/prerelease-testing, prerelease-testing) standardize on main/preview/release naming
|
||||
- 5facaddfe actions testing
|
||||
- 33c55809d actions testing
|
||||
- fcdddf307 simplify version
|
||||
- ac7f5762c actions cleanup
|
||||
|
@ -50,7 +50,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.tinkerpop</groupId>
|
||||
<artifactId>gremlin-core</artifactId>
|
||||
<version>3.6.1</version>
|
||||
<version>3.6.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.tinkerpop</groupId>
|
||||
@ -140,48 +140,6 @@
|
||||
<followSymLinks>false</followSymLinks>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
<resources>
|
||||
<resource>
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,13 +16,13 @@
|
||||
|
||||
package io.nosqlbench.cqlgen.core;
|
||||
|
||||
import io.nosqlbench.cqlgen.model.CqlColumnBase;
|
||||
import io.nosqlbench.cqlgen.binders.Binding;
|
||||
import io.nosqlbench.cqlgen.api.BindingsLibrary;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.api.content.Content;
|
||||
import io.nosqlbench.api.content.NBIO;
|
||||
import io.nosqlbench.cqlgen.api.BindingsLibrary;
|
||||
import io.nosqlbench.cqlgen.binders.Binding;
|
||||
import io.nosqlbench.cqlgen.model.CqlColumnBase;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
@ -36,14 +36,17 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
|
||||
public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml";
|
||||
|
||||
public CGDefaultCqlBindings() {
|
||||
String yamlContent = NBIO.all()
|
||||
Content<?> content = NBIO.all()
|
||||
.name(DEFAULT_BINDINGS_FILE)
|
||||
.first()
|
||||
.map(Content::asString)
|
||||
.or(() -> loadLocal(DEFAULT_BINDINGS_FILE))
|
||||
.orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE));
|
||||
StmtsDocList stmtsDocs = StatementsLoader.loadString(yamlContent, Map.of());
|
||||
this.bindings = stmtsDocs.getDocBindings();
|
||||
.or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first())
|
||||
.orElseThrow(
|
||||
() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE +
|
||||
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)
|
||||
);
|
||||
|
||||
OpsDocList opsDocList = OpsLoader.loadContent(content, Map.of());
|
||||
this.bindings = opsDocList.getDocBindings();
|
||||
}
|
||||
|
||||
private Optional<String> loadLocal(String path) {
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -19,7 +19,7 @@ package io.nosqlbench.cqlgen.transformers;
|
||||
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
|
||||
import io.nosqlbench.cqlgen.core.CGSchemaStats;
|
||||
import io.nosqlbench.cqlgen.core.CGTableStats;
|
||||
import org.apache.commons.math4.util.Pair;
|
||||
import org.apache.commons.math4.legacy.core.Pair;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
|
@ -6,13 +6,13 @@ description: |
|
||||
|
||||
scenarios:
|
||||
default:
|
||||
schema: run driver=cql tags==phase:schema threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
astra:
|
||||
schema: run driver=cql tags==phase:schema-astra threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
schema: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
|
||||
params:
|
||||
instrument: TEMPLATE(instrument,false)
|
||||
@ -20,7 +20,7 @@ params:
|
||||
bindings:
|
||||
machine_id: Mod(<<sources:10000>>); ToHashedUUID() -> java.util.UUID
|
||||
sensor_name: HashedLineToString('data/variable_words.txt')
|
||||
time: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); ToDate()
|
||||
time: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); ToJavaInstant();
|
||||
cell_timestamp: Mul(<<timespeed:100>>L); Div(<<sources:10000>>L); Mul(1000L)
|
||||
sensor_value: Normal(0.0,5.0); Add(100.0) -> double
|
||||
station_id: Div(<<sources:10000>>);Mod(<<stations:100>>); ToHashedUUID() -> java.util.UUID
|
||||
|
@ -1,14 +1,22 @@
|
||||
description: A workload with only text keys and text values
|
||||
description: |
|
||||
A workload with only text keys and text values.
|
||||
This workload is rather simple but it is useful for demonstration purposes.
|
||||
If you want a more thorough example, you might be interested in the
|
||||
baselinesv2/cql-* workloads.
|
||||
defaults:
|
||||
TEMPLATE(keyspace,baselines)
|
||||
TEMPLATE(table,keyvalue)
|
||||
|
||||
|
||||
scenarios:
|
||||
default:
|
||||
schema: run driver=cql tags==phase:schema threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags=='block:main-.*' cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
astra:
|
||||
schema: run driver=cql tags==phase:schema-astra threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
schema: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF
|
||||
rampup: run driver=cql tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
|
||||
main: run driver=cql tags=='block:main-.*' cycles===TEMPLATE(main-cycles,10000000) threads=auto
|
||||
|
||||
bindings:
|
||||
seq_key: Mod(<<keycount:1000000000>>); ToString() -> String
|
||||
@ -17,86 +25,60 @@ bindings:
|
||||
rw_value: Hash(); <<valdist:Uniform(0,1000000000)->int>>; ToString() -> String
|
||||
|
||||
blocks:
|
||||
- name: schema
|
||||
tags:
|
||||
phase: schema
|
||||
params:
|
||||
prepared: false
|
||||
statements:
|
||||
- create-keyspace: |
|
||||
create keyspace if not exists <<keyspace:baselines>>
|
||||
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '<<rf:1>>'}
|
||||
AND durable_writes = true;
|
||||
tags:
|
||||
name: create-keyspace
|
||||
- create-table: |
|
||||
create table if not exists <<keyspace:baselines>>.<<table:keyvalue>> (
|
||||
schema:
|
||||
ops:
|
||||
create-keyspace:
|
||||
raw: |
|
||||
create keyspace if not exists TEMPLATE(keyspace,baselines)
|
||||
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 'TEMPLATE(rf,1)>>'}
|
||||
AND durable_writes = true;
|
||||
create-table:
|
||||
raw: |
|
||||
create table if not exists TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) (
|
||||
key text,
|
||||
value text,
|
||||
PRIMARY KEY (key)
|
||||
);
|
||||
schema-astra:
|
||||
ops:
|
||||
create-table:
|
||||
raw: |
|
||||
create table if not exists TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) (
|
||||
key text,
|
||||
value text,
|
||||
PRIMARY KEY (key)
|
||||
);
|
||||
tags:
|
||||
name: create-table
|
||||
- name: schema-astra
|
||||
tags:
|
||||
phase: schema-astra
|
||||
rampup:
|
||||
params:
|
||||
prepared: false
|
||||
statements:
|
||||
- create-table: |
|
||||
create table if not exists <<keyspace:baselines>>.<<table:keyvalue>> (
|
||||
key text,
|
||||
value text,
|
||||
PRIMARY KEY (key)
|
||||
);
|
||||
tags:
|
||||
name: create-table-astra
|
||||
- name: rampup
|
||||
tags:
|
||||
phase: rampup
|
||||
params:
|
||||
cl: <<write_cl:LOCAL_QUORUM>>
|
||||
statements:
|
||||
- rampup-insert: |
|
||||
insert into <<keyspace:baselines>>.<<table:keyvalue>>
|
||||
cl: TEMPLATE(write_cl,LOCAL_QUORUM)
|
||||
ops:
|
||||
rampup-insert:
|
||||
prepared: |
|
||||
insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
|
||||
(key, value)
|
||||
values ({seq_key},{seq_value});
|
||||
tags:
|
||||
name: rampup-insert
|
||||
- name: verify
|
||||
tags:
|
||||
phase: verify
|
||||
type: read
|
||||
params:
|
||||
cl: <<read_cl:LOCAL_QUORUM>>
|
||||
statements:
|
||||
- verify-select: |
|
||||
select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={seq_key};
|
||||
verify-fields: key->seq_key, value->seq_value
|
||||
tags:
|
||||
name: verify
|
||||
- name: main-read
|
||||
tags:
|
||||
phase: main
|
||||
type: read
|
||||
# verify:
|
||||
# params:
|
||||
# cl: <<read_cl:LOCAL_QUORUM>>
|
||||
# ops:
|
||||
# verify-select:
|
||||
# prepared: |
|
||||
# select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={seq_key};
|
||||
# verify-fields: key->seq_key, value->seq_value
|
||||
main-read:
|
||||
params:
|
||||
ratio: 5
|
||||
cl: <<read_cl:LOCAL_QUORUM>>
|
||||
statements:
|
||||
- main-select: |
|
||||
select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={rw_key};
|
||||
tags:
|
||||
name: main-select
|
||||
- name: main-write
|
||||
tags:
|
||||
phase: main
|
||||
type: write
|
||||
cl: TEMPLATE(read_cl,LOCAL_QUORUM)
|
||||
ops:
|
||||
main-select:
|
||||
prepared: |
|
||||
select * from TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue) where key={rw_key};
|
||||
main-write:
|
||||
params:
|
||||
ratio: 5
|
||||
cl: <<write_cl:LOCAL_QUORUM>>
|
||||
statements:
|
||||
- main-insert: |
|
||||
insert into <<keyspace:baselines>>.<<table:keyvalue>>
|
||||
cl: TEMPLATE(write_cl,LOCAL_QUORUM)
|
||||
ops:
|
||||
main-insert:
|
||||
prepared: |
|
||||
insert into TEMPLATE(keyspace,baselines).TEMPLATE(table,keyvalue)
|
||||
(key, value) values ({rw_key}, {rw_value});
|
||||
tags:
|
||||
name: main-insert
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -17,9 +17,10 @@
|
||||
package io.nosqlbench.adapter.diag;
|
||||
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
|
||||
@ -101,7 +102,7 @@ public class DiagDriverAdapter extends BaseDriverAdapter<DiagOp, DiagSpace> impl
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> params) {
|
||||
return StatementsLoader.loadString("ops: 'log:level=INFO'", params).getStmts();
|
||||
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params) {
|
||||
return OpsLoader.loadString("log:level=INFO", OpTemplateFormat.inline, params,null).getOps();
|
||||
}
|
||||
}
|
||||
|
@ -48,53 +48,4 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
</project>
|
||||
|
@ -33,53 +33,6 @@
|
||||
This provides the ability to generate HTTP requests with nosqlbench
|
||||
</description>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -19,9 +19,10 @@ package io.nosqlbench.adapter.http;
|
||||
import io.nosqlbench.adapter.http.core.HttpOpMapper;
|
||||
import io.nosqlbench.adapter.http.core.HttpSpace;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
@ -51,9 +52,9 @@ public class HttpOpMapperTest {
|
||||
}
|
||||
|
||||
private static ParsedOp parsedOpFor(String yaml) {
|
||||
StmtsDocList docs = StatementsLoader.loadString(yaml, Map.of());
|
||||
OpTemplate stmtDef = docs.getStmts().get(0);
|
||||
ParsedOp parsedOp = new ParsedOp(stmtDef, cfg, List.of(adapter.getPreprocessor()));
|
||||
OpsDocList docs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
|
||||
OpTemplate opTemplate = docs.getOps().get(0);
|
||||
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg, List.of(adapter.getPreprocessor()));
|
||||
return parsedOp;
|
||||
}
|
||||
|
||||
@ -101,7 +102,7 @@ public class HttpOpMapperTest {
|
||||
@Test
|
||||
public void testRFCFormBody() {
|
||||
ParsedOp pop = parsedOpFor("""
|
||||
statements:
|
||||
ops:
|
||||
- s1: |
|
||||
get http://localhost/
|
||||
|
||||
@ -117,7 +118,7 @@ public class HttpOpMapperTest {
|
||||
// This can not be fully resolved in the unit testing context, but it could be
|
||||
// in the integrated testing context. It is sufficient to verify parsing here.
|
||||
ParsedOp pop = parsedOpFor("""
|
||||
statements:
|
||||
ops:
|
||||
- s1: |
|
||||
{method} {scheme}://{host}/{path}?{query} {version}
|
||||
Header1: {header1val}
|
||||
|
@ -34,55 +34,9 @@
|
||||
</description>
|
||||
|
||||
<properties>
|
||||
<kafka.version>3.3.1</kafka.version>
|
||||
<kafka.version>3.3.2</kafka.version>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.nosqlbench</groupId>
|
||||
|
@ -46,50 +46,4 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
@ -91,51 +91,4 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
@ -100,50 +100,6 @@
|
||||
<version>1.7.1</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
@ -53,50 +53,9 @@
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -18,7 +18,7 @@ package io.nosqlbench.adapter.stdout;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
|
||||
@ -59,8 +59,8 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
|
||||
Set<String> activeBindingNames = new LinkedHashSet<>(stmtsDocList.getDocBindings().keySet());
|
||||
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
|
||||
Set<String> activeBindingNames = new LinkedHashSet<>(opsDocList.getDocBindings().keySet());
|
||||
|
||||
if (activeBindingNames.size()==0) {
|
||||
logger.warn("Unable to synthesize op for driver=" + this.getAdapterName() + " with zero bindings.");
|
||||
@ -88,7 +88,7 @@ public class StdoutDriverAdapter extends BaseDriverAdapter<StdoutOp, StdoutSpace
|
||||
return List.of();
|
||||
}
|
||||
|
||||
OpData op = new OpData("synthetic", "synthetic", Map.of(), stmtsDocList.getDocBindings(), cfg,
|
||||
OpData op = new OpData("synthetic", "synthetic", Map.of(), opsDocList.getDocBindings(), cfg,
|
||||
Map.of("stmt", genStatementTemplate(filteredBindingNames, cfg)));
|
||||
|
||||
return List.of(op);
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -14,12 +14,26 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* This package is an import of some org.apache.commons source trees.
|
||||
* Once the state of the newer o.a.c.math4 and distributions modules is stable,
|
||||
* these will be removed and the release artifacts will be used.
|
||||
* The enclosed packages were included here as the least worst of several options,
|
||||
* given the confluence of JPMS, the status of the math4 repos, and the need
|
||||
* to access stable CDF functions across the extant distributions.
|
||||
*/
|
||||
package org.apache.commons;
|
||||
package io.nosqlbench.adapter.stdout;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class TemplateFormatTest {
|
||||
|
||||
@Test
|
||||
public void testBasicFormats() {
|
||||
assertThat(TemplateFormat.json.format(true, List.of("f1","f2")))
|
||||
.isEqualTo("""
|
||||
{
|
||||
"f1":"{f1}",
|
||||
"f2":"{f2}"
|
||||
}
|
||||
|
||||
""");
|
||||
}
|
||||
|
||||
}
|
@ -51,51 +51,4 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -21,7 +21,7 @@ import io.nosqlbench.api.config.standard.ConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
|
||||
@ -58,8 +58,8 @@ public class TcpClientDriverAdapter extends BaseDriverAdapter<TcpClientOp, TcpCl
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
|
||||
return adap.getSyntheticOpTemplates(stmtsDocList, cfg);
|
||||
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> cfg) {
|
||||
return adap.getSyntheticOpTemplates(opsDocList, cfg);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -20,7 +20,7 @@ import io.nosqlbench.api.config.standard.ConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
|
||||
@ -57,7 +57,7 @@ public class TcpServerDriverAdapter extends BaseDriverAdapter<TcpServerOp, TcpSe
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> cfg) {
|
||||
return adap.getSyntheticOpTemplates(stmtsDocList, cfg);
|
||||
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String,Object> cfg) {
|
||||
return adap.getSyntheticOpTemplates(opsDocList, cfg);
|
||||
}
|
||||
}
|
||||
|
@ -54,54 +54,21 @@
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.databricks</groupId>
|
||||
<artifactId>sjsonnet_2.13</artifactId>
|
||||
<version>0.4.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-library</artifactId>
|
||||
<version>2.13.10</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
|
@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig;
|
||||
|
||||
import com.amazonaws.util.StringInputStream;
|
||||
import io.nosqlbench.api.content.Content;
|
||||
import io.nosqlbench.api.content.NBIO;
|
||||
import io.nosqlbench.api.errors.BasicError;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.templating.StrInterpolator;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import scala.Option;
|
||||
import sjsonnet.DefaultParseCache;
|
||||
import sjsonnet.SjsonnetMain;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class OpsLoader {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(OpsLoader.class);
|
||||
|
||||
public static String[] YAML_EXTENSIONS = new String[]{"yaml", "yml"};
|
||||
|
||||
public static OpsDocList loadContent(Content<?> content, Map<String, String> params) {
|
||||
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(content.getURI());
|
||||
return loadString(content.get().toString(), fmt, params, content.getURI());
|
||||
}
|
||||
|
||||
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
|
||||
String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS;
|
||||
|
||||
Content<?> foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first()
|
||||
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
|
||||
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
|
||||
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());
|
||||
}
|
||||
|
||||
public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map<String, ?> params, URI srcuri) {
|
||||
|
||||
logger.trace(() -> "Applying string transformer to data:" + sourceData);
|
||||
StrInterpolator transformer = new StrInterpolator(params);
|
||||
String data = transformer.apply(sourceData);
|
||||
if (srcuri!=null) {
|
||||
logger.info("workload URI: '" + srcuri + "'");
|
||||
}
|
||||
|
||||
RawOpsLoader loader = new RawOpsLoader(transformer);
|
||||
RawOpsDocList rawOpsDocList = switch (fmt) {
|
||||
case jsonnet -> loader.loadString(evaluateJsonnet(srcuri, params));
|
||||
case yaml, json -> loader.loadString(data);
|
||||
case inline, stmt -> RawOpsDocList.forSingleStatement(data);
|
||||
};
|
||||
// TODO: itemize inline to support ParamParser
|
||||
|
||||
OpsDocList layered = new OpsDocList(rawOpsDocList);
|
||||
|
||||
transformer.checkpointAccesses().forEach((k, v) -> {
|
||||
layered.addTemplateVariable(k, v);
|
||||
if (params.containsKey(k)) {
|
||||
params.remove(k);
|
||||
}
|
||||
});
|
||||
|
||||
return layered;
|
||||
}
|
||||
|
||||
private static String evaluateJsonnet(URI uri, Map<String, ?> params) {
|
||||
List<String> injected = new LinkedList<>(List.of(Path.of(uri).toString()));
|
||||
params.forEach((k,v) -> {
|
||||
if (v instanceof CharSequence cs) {
|
||||
injected.addAll(List.of("--ext-str",k+"="+cs));
|
||||
}
|
||||
});
|
||||
|
||||
var stdoutBuffer = new ByteArrayOutputStream();
|
||||
var stderrBuffer = new ByteArrayOutputStream();
|
||||
var stdoutStream = new PrintStream(stdoutBuffer);
|
||||
var stderrStream = new PrintStream(stderrBuffer);
|
||||
StringInputStream inputStream;
|
||||
try {
|
||||
inputStream = new StringInputStream("");
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Error building input stream for jsonnet:" + e, e);
|
||||
}
|
||||
|
||||
int resultStatus = SjsonnetMain.main0(
|
||||
injected.toArray(new String[0]),
|
||||
new DefaultParseCache(),
|
||||
inputStream,
|
||||
stdoutStream,
|
||||
stderrStream,
|
||||
new os.Path(Path.of(System.getProperty("user.dir"))),
|
||||
Option.empty(),
|
||||
Option.empty()
|
||||
);
|
||||
|
||||
String stdoutOutput = stdoutBuffer.toString(StandardCharsets.UTF_8);
|
||||
String stderrOutput = stderrBuffer.toString(StandardCharsets.UTF_8);
|
||||
if ("jsonnet".equals(String.valueOf(params.get("dryrun")))) {
|
||||
logger.info("dryrun=jsonnet, dumping result to stdout and stderr:");
|
||||
System.out.println(stdoutOutput);
|
||||
System.err.println(stderrOutput);
|
||||
if (resultStatus==0 && stderrOutput.isEmpty()) {
|
||||
logger.info("no errors detected during jsonnet evaluation.");
|
||||
System.exit(0);
|
||||
} else {
|
||||
logger.error("ERRORS detected during jsonnet evaluation:\n" + stderrOutput);
|
||||
System.exit(2);
|
||||
}
|
||||
}
|
||||
if (!stderrOutput.isEmpty()) {
|
||||
BasicError error = new BasicError("stderr output from jsonnet preprocessing: " + stderrOutput);
|
||||
if (resultStatus!=0) {
|
||||
throw error;
|
||||
} else {
|
||||
logger.warn(error.toString(),error);
|
||||
}
|
||||
}
|
||||
logger.info("jsonnet processing read '" + uri +"', rendered " + stdoutOutput.split("\n").length + " lines.");
|
||||
logger.trace("jsonnet result:\n" + stdoutOutput);
|
||||
|
||||
return stdoutOutput;
|
||||
}
|
||||
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -27,7 +27,7 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
|
||||
public class ParsedStmtOp {
|
||||
public class ParsedOpTemplate {
|
||||
|
||||
private final OpTemplate optpl;
|
||||
private final ParsedStringTemplate parsed;
|
||||
@ -37,13 +37,13 @@ public class ParsedStmtOp {
|
||||
*
|
||||
* @param optpl An existing statement def as read from the YAML API.
|
||||
*/
|
||||
public ParsedStmtOp(OpTemplate optpl) {
|
||||
public ParsedOpTemplate(OpTemplate optpl) {
|
||||
this.optpl = optpl;
|
||||
String transformed = getStmt();
|
||||
parsed = new ParsedStringTemplate(transformed, optpl.getBindings());
|
||||
}
|
||||
|
||||
public ParsedStmtOp orError() {
|
||||
public ParsedOpTemplate orError() {
|
||||
if (hasError()) {
|
||||
throw new RuntimeException("Unable to parse statement: " + this);
|
||||
}
|
@ -1,99 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.templating.StrInterpolator;
|
||||
import io.nosqlbench.api.content.Content;
|
||||
import io.nosqlbench.api.content.NBIO;
|
||||
import io.nosqlbench.api.errors.BasicError;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
public class StatementsLoader {
|
||||
|
||||
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(StatementsLoader.class);
|
||||
|
||||
public static StmtsDocList loadString(String yamlContent, Map<String,?> params) {
|
||||
|
||||
StrInterpolator transformer = new StrInterpolator(params);
|
||||
RawStmtsLoader loader = new RawStmtsLoader(transformer);
|
||||
RawStmtsDocList rawDocList = loader.loadString(logger, yamlContent);
|
||||
StmtsDocList layered = new StmtsDocList(rawDocList);
|
||||
transformer.checkpointAccesses().forEach((k,v) -> {
|
||||
layered.addTemplateVariable(k,v);
|
||||
if (params.containsKey(k)) {
|
||||
params.remove(k);
|
||||
}
|
||||
});
|
||||
return layered;
|
||||
}
|
||||
|
||||
public static StmtsDocList loadStmt(
|
||||
Logger logger,
|
||||
String statement,
|
||||
Map<String,?> params
|
||||
) {
|
||||
StrInterpolator transformer = new StrInterpolator(params);
|
||||
statement = transformer.apply(statement);
|
||||
RawStmtsDocList rawStmtsDocList = RawStmtsDocList.forSingleStatement(statement);
|
||||
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
|
||||
transformer.checkpointAccesses().forEach((k,v) -> {
|
||||
layered.addTemplateVariable(k,v);
|
||||
if (params.containsKey(k)) {
|
||||
params.remove(k);
|
||||
}
|
||||
});
|
||||
return layered;
|
||||
}
|
||||
|
||||
public static StmtsDocList loadContent(
|
||||
Logger logger,
|
||||
Content<?> content,
|
||||
Map<String,String> params
|
||||
) {
|
||||
return loadString(content.get().toString(),params);
|
||||
}
|
||||
|
||||
public static StmtsDocList loadPath(
|
||||
Logger logger,
|
||||
String path,
|
||||
Map<String,?> params,
|
||||
String... searchPaths) {
|
||||
|
||||
RawStmtsDocList list = null;
|
||||
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
|
||||
String content = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
|
||||
return loadString(content,params);
|
||||
}
|
||||
|
||||
|
||||
public static StmtsDocList loadPath(
|
||||
Logger logger,
|
||||
String path,
|
||||
String... searchPaths) {
|
||||
return loadPath(logger, path, Map.of(), searchPaths);
|
||||
}
|
||||
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -20,21 +20,21 @@ import io.nosqlbench.api.errors.BasicError;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class StatementsOwner extends RawStmtFields {
|
||||
public class OpsOwner extends RawOpFields {
|
||||
|
||||
private final static List<String> stmtsFieldNames = List.of("op","ops","operation","statement","statements");
|
||||
private final static List<String> opsFieldNames = List.of("op","ops","operation","statement","statements");
|
||||
|
||||
private List<RawStmtDef> rawStmtDefs = new ArrayList<>();
|
||||
private List<RawOpDef> rawOpDefs = new ArrayList<>();
|
||||
|
||||
public StatementsOwner() {
|
||||
public OpsOwner() {
|
||||
}
|
||||
|
||||
public List<RawStmtDef> getRawStmtDefs() {
|
||||
return rawStmtDefs;
|
||||
public List<RawOpDef> getRawOpDefs() {
|
||||
return rawOpDefs;
|
||||
}
|
||||
|
||||
public void setRawStmtDefs(List<RawStmtDef> rawStmtDefs) {
|
||||
this.rawStmtDefs = rawStmtDefs;
|
||||
public void setRawStmtDefs(List<RawOpDef> rawOpDefs) {
|
||||
this.rawOpDefs = rawOpDefs;
|
||||
}
|
||||
|
||||
public void setFieldsByReflection(Map<String, Object> propsmap) {
|
||||
@ -43,7 +43,7 @@ public class StatementsOwner extends RawStmtFields {
|
||||
super.setFieldsByReflection(propsmap);
|
||||
|
||||
HashSet<String> found = new HashSet<>();
|
||||
for (String fname : stmtsFieldNames) {
|
||||
for (String fname : opsFieldNames) {
|
||||
if (propsmap.containsKey(fname)) {
|
||||
found.add(fname);
|
||||
}
|
||||
@ -52,25 +52,25 @@ public class StatementsOwner extends RawStmtFields {
|
||||
throw new BasicError("You used " + found + " as an op name, but only one of these is allowed.");
|
||||
}
|
||||
if (found.size()==1) {
|
||||
Object stmtsFieldValue = propsmap.remove(found.iterator().next());
|
||||
setStatementsFieldByType(stmtsFieldValue);
|
||||
Object opsFieldValue = propsmap.remove(found.iterator().next());
|
||||
setOpsFieldByType(opsFieldValue);
|
||||
}
|
||||
|
||||
super.setFieldsByReflection(propsmap);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void setStatementsFieldByType(Object object) {
|
||||
public void setOpsFieldByType(Object object) {
|
||||
if (object instanceof List) {
|
||||
List<Object> stmtList = (List<Object>) object;
|
||||
List<RawStmtDef> defs = new ArrayList<>(stmtList.size());
|
||||
List<RawOpDef> defs = new ArrayList<>(stmtList.size());
|
||||
for (int i = 0; i < stmtList.size(); i++) {
|
||||
String defaultName = "stmt" + (i + 1);
|
||||
Object o = stmtList.get(i);
|
||||
if (o instanceof String) {
|
||||
defs.add(new RawStmtDef(defaultName, (String) o));
|
||||
defs.add(new RawOpDef(defaultName, (String) o));
|
||||
} else if (o instanceof Map) {
|
||||
RawStmtDef def = new RawStmtDef(defaultName, (Map<String, Object>) o);
|
||||
RawOpDef def = new RawOpDef(defaultName, (Map<String, Object>) o);
|
||||
defs.add(def);
|
||||
} else {
|
||||
throw new RuntimeException("Can not construct stmt def from object type:" + o.getClass());
|
||||
@ -105,9 +105,9 @@ public class StatementsOwner extends RawStmtFields {
|
||||
entries.getValue().getClass() + "', only maps and strings are recognized.");
|
||||
}
|
||||
}
|
||||
setStatementsFieldByType(itemizedMaps);
|
||||
setOpsFieldByType(itemizedMaps);
|
||||
} else if (object instanceof String) {
|
||||
setStatementsFieldByType(Map.of("stmt1", (String) object));
|
||||
setOpsFieldByType(Map.of("stmt1", (String) object));
|
||||
} else {
|
||||
throw new RuntimeException("Unknown object type: " + object.getClass());
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -24,22 +24,22 @@ import java.util.*;
|
||||
/**
|
||||
* See specification for what this should do in UniformWorkloadSpecificationTest
|
||||
*/
|
||||
public class RawStmtDef extends RawStmtFields {
|
||||
public class RawOpDef extends RawOpFields {
|
||||
|
||||
private Object op;
|
||||
|
||||
private final static List<String> opFieldSynonyms = List.of("stmt", "statement", "op", "operation");
|
||||
|
||||
public RawStmtDef() {
|
||||
public RawOpDef() {
|
||||
}
|
||||
|
||||
public RawStmtDef(String name, String op) {
|
||||
public RawOpDef(String name, String op) {
|
||||
setName(name);
|
||||
this.op = op;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public RawStmtDef(String defaultName, Map<String, Object> map) {
|
||||
public RawOpDef(String defaultName, Map<String, Object> map) {
|
||||
setFieldsByReflection(map);
|
||||
if (this.getName() == null || this.getName().isEmpty()) {
|
||||
this.setName(defaultName);
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -25,15 +25,15 @@ import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
public class RawStmtFields extends Tags {
|
||||
private final static Logger logger = LogManager.getLogger(RawStmtFields.class);
|
||||
public class RawOpFields extends Tags {
|
||||
private final static Logger logger = LogManager.getLogger(RawOpFields.class);
|
||||
|
||||
private String name = "";
|
||||
private String desc = "";
|
||||
private final Map<String, String> bindings = new LinkedHashMap<>();
|
||||
private final Map<String, Object> params = new LinkedHashMap<>();
|
||||
|
||||
public RawStmtFields() {
|
||||
public RawOpFields() {
|
||||
}
|
||||
|
||||
public String getDesc() {
|
||||
@ -80,7 +80,7 @@ public class RawStmtFields extends Tags {
|
||||
this.params.putAll(config);
|
||||
}
|
||||
|
||||
public void applyBlockParams(RawStmtFields other) {
|
||||
public void applyBlockParams(RawOpFields other) {
|
||||
setName(other.getName());
|
||||
setBindings(other.getBindings());
|
||||
setTags(other.getTags());
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -20,12 +20,12 @@ import java.util.Map;
|
||||
|
||||
/**
|
||||
* A StmtsDef contains a list of rawStmts, as well as all of the optional
|
||||
* block parameters that can be assigned to {@link RawStmtFields}, which includes
|
||||
* block parameters that can be assigned to {@link RawOpFields}, which includes
|
||||
* a name, config values, data bindings, and filtering tags.
|
||||
*/
|
||||
public class RawStmtsBlock extends StatementsOwner {
|
||||
public class RawOpsBlock extends OpsOwner {
|
||||
|
||||
public RawStmtsBlock() {}
|
||||
public RawOpsBlock() {}
|
||||
|
||||
public void setFieldsByReflection(Map<String, Object> propsmap) {
|
||||
super.setFieldsByReflection(propsmap);
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -22,29 +22,20 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A statements doc can have both a list of statement blocks and/or a
|
||||
* list of statements. It can also have all the block parameters
|
||||
* assignable to {@link RawStmtFields}.
|
||||
* <p>
|
||||
* The reason for having support both statements or statement blocks
|
||||
* is merely convenience. If you do not need or want to deal with the
|
||||
* full blocks format, the extra structure gets in the way.
|
||||
*/
|
||||
public class RawStmtsDoc extends StatementsOwner {
|
||||
public class RawOpsDoc extends OpsOwner {
|
||||
|
||||
private RawScenarios scenarios = new RawScenarios();
|
||||
private final List<RawStmtsBlock> blocks = new ArrayList<>();
|
||||
private final List<RawOpsBlock> blocks = new ArrayList<>();
|
||||
private String versionRegex = ".+";
|
||||
|
||||
// no-args ctor is required
|
||||
public RawStmtsDoc() {
|
||||
public RawOpsDoc() {
|
||||
}
|
||||
|
||||
public static RawStmtsDoc forSingleStatement(String statement) {
|
||||
RawStmtsDoc rawStmtsDoc = new RawStmtsDoc();
|
||||
rawStmtsDoc.setStatementsFieldByType(statement);
|
||||
return rawStmtsDoc;
|
||||
public static RawOpsDoc forSingleStatement(String statement) {
|
||||
RawOpsDoc rawOpsDoc = new RawOpsDoc();
|
||||
rawOpsDoc.setOpsFieldByType(statement);
|
||||
return rawOpsDoc;
|
||||
}
|
||||
|
||||
public void setFieldsByReflection(Map<String, Object> properties) {
|
||||
@ -63,9 +54,9 @@ public class RawStmtsDoc extends StatementsOwner {
|
||||
for (Object blockData : blockList) {
|
||||
if (blockData instanceof Map) {
|
||||
Map<String, Object> blockDataMap = (Map<String, Object>) blockData;
|
||||
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
|
||||
rawStmtsBlock.setFieldsByReflection(blockDataMap);
|
||||
blocks.add(rawStmtsBlock);
|
||||
RawOpsBlock rawOpsBlock = new RawOpsBlock();
|
||||
rawOpsBlock.setFieldsByReflection(blockDataMap);
|
||||
blocks.add(rawOpsBlock);
|
||||
} else {
|
||||
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
|
||||
}
|
||||
@ -77,10 +68,10 @@ public class RawStmtsDoc extends StatementsOwner {
|
||||
Object blockData = entry.getValue();
|
||||
if (blockData instanceof Map) {
|
||||
Map<String, Object> blockDataMap = (Map<String, Object>) blockData;
|
||||
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
|
||||
rawStmtsBlock.setName(blockName);
|
||||
rawStmtsBlock.setFieldsByReflection(blockDataMap);
|
||||
blocks.add(rawStmtsBlock);
|
||||
RawOpsBlock rawOpsBlock = new RawOpsBlock();
|
||||
rawOpsBlock.setName(blockName);
|
||||
rawOpsBlock.setFieldsByReflection(blockDataMap);
|
||||
blocks.add(rawOpsBlock);
|
||||
} else {
|
||||
throw new RuntimeException("Invalid object type for block data: " + blockData.getClass().getCanonicalName());
|
||||
}
|
||||
@ -100,26 +91,24 @@ public class RawStmtsDoc extends StatementsOwner {
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the list of statement blocks in this RawStmtsDoc.
|
||||
* If raw statements are defined on this RawStmtsDoc, then a single
|
||||
* StmtBlock containing those statements is prepended to the block list.
|
||||
* Otherwise, the list of StmtBlocks is returned as-is.
|
||||
* If raw ops are defined on this RawOpsDoc, then a single RawOpsBlock containing those op templates is prepended to
|
||||
* the block list. Otherwise, the list of RawOpsBlocks is returned as-is.
|
||||
*
|
||||
* @return all logical statement blocks containing statements
|
||||
* @return all logical ops blocks
|
||||
*/
|
||||
public List<RawStmtsBlock> getBlocks() {
|
||||
List<RawStmtsBlock> stmtBlocks = new ArrayList<>();
|
||||
if (!getRawStmtDefs().isEmpty()) {
|
||||
RawStmtsBlock rawStmtsBlock = new RawStmtsBlock();
|
||||
rawStmtsBlock.setName("block0");
|
||||
rawStmtsBlock.setRawStmtDefs(getRawStmtDefs());
|
||||
stmtBlocks.add(rawStmtsBlock);
|
||||
public List<RawOpsBlock> getBlocks() {
|
||||
List<RawOpsBlock> stmtBlocks = new ArrayList<>();
|
||||
if (!getRawOpDefs().isEmpty()) {
|
||||
RawOpsBlock rawOpsBlock = new RawOpsBlock();
|
||||
rawOpsBlock.setName("block0");
|
||||
rawOpsBlock.setRawStmtDefs(getRawOpDefs());
|
||||
stmtBlocks.add(rawOpsBlock);
|
||||
}
|
||||
stmtBlocks.addAll(this.blocks);
|
||||
return stmtBlocks;
|
||||
}
|
||||
|
||||
public void setBlocks(List<RawStmtsBlock> blocks) {
|
||||
public void setBlocks(List<RawOpsBlock> blocks) {
|
||||
this.blocks.clear();
|
||||
this.blocks.addAll(blocks);
|
||||
}
|
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.rawyaml;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class RawOpsDocList {
|
||||
|
||||
private final List<RawOpsDoc> rawOpsDocList;
|
||||
|
||||
public RawOpsDocList(List<RawOpsDoc> rawOpsDocList) {
|
||||
this.rawOpsDocList = rawOpsDocList;
|
||||
}
|
||||
|
||||
public static RawOpsDocList forSingleStatement(String statement) {
|
||||
RawOpsDoc rawOpsDoc = RawOpsDoc.forSingleStatement(statement);
|
||||
return new RawOpsDocList(List.of(rawOpsDoc));
|
||||
}
|
||||
|
||||
public static RawOpsDocList none() {
|
||||
return new RawOpsDocList(List.of());
|
||||
}
|
||||
|
||||
public List<RawOpsDoc> getOpsDocs() {
|
||||
return rawOpsDocList;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
int docs = rawOpsDocList.size();
|
||||
int blocks = rawOpsDocList.stream().map(RawOpsDoc::getBlocks).mapToInt(List::size).sum();
|
||||
long optemplates = rawOpsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawOpDefs().stream()).count();
|
||||
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
|
||||
}
|
||||
}
|
@ -21,6 +21,7 @@ import io.nosqlbench.api.content.NBIO;
|
||||
import io.nosqlbench.api.errors.BasicError;
|
||||
import io.nosqlbench.api.errors.OpConfigError;
|
||||
import io.nosqlbench.engine.api.templating.StrInterpolator;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.snakeyaml.engine.v2.api.Load;
|
||||
import org.snakeyaml.engine.v2.api.LoadSettings;
|
||||
@ -28,18 +29,18 @@ import org.snakeyaml.engine.v2.api.LoadSettings;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
|
||||
public class RawStmtsLoader {
|
||||
public class RawOpsLoader {
|
||||
private final static Logger logger = LogManager.getLogger(RawOpsLoader.class);
|
||||
|
||||
public static String[] YAML_EXTENSIONS = new String[]{"yaml","yml"};
|
||||
|
||||
List<Function<String, String>> stringTransformers = new ArrayList<>();
|
||||
private final ArrayList<Function<String,String>> transformers = new ArrayList<>();
|
||||
|
||||
public RawStmtsLoader(Function<String,String> transformer) {
|
||||
public RawOpsLoader(Function<String,String> transformer) {
|
||||
addTransformer(transformer);
|
||||
}
|
||||
|
||||
public RawStmtsLoader() {
|
||||
public RawOpsLoader() {
|
||||
addTransformer(new StrInterpolator());
|
||||
}
|
||||
|
||||
@ -47,11 +48,10 @@ public class RawStmtsLoader {
|
||||
Collections.addAll(this.transformers, newTransformer);
|
||||
}
|
||||
|
||||
public RawStmtsDocList loadString(Logger logger, final String originalData) {
|
||||
public RawOpsDocList loadString(final String originalData) {
|
||||
logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
|
||||
String data = originalData;
|
||||
|
||||
try {
|
||||
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + originalData);
|
||||
for (Function<String, String> transformer : transformers) {
|
||||
data = transformer.apply(data);
|
||||
}
|
||||
@ -60,11 +60,10 @@ public class RawStmtsLoader {
|
||||
throw t;
|
||||
}
|
||||
|
||||
return parseYaml(logger, data);
|
||||
return parseYaml(data);
|
||||
}
|
||||
|
||||
public RawStmtsDocList loadPath(
|
||||
Logger logger,
|
||||
public RawOpsDocList loadPath(
|
||||
String path,
|
||||
String... searchPaths) {
|
||||
|
||||
@ -72,22 +71,22 @@ public class RawStmtsLoader {
|
||||
try {
|
||||
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
|
||||
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
|
||||
return loadString(logger, data);
|
||||
return loadString(data);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("error while reading file " + path, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private RawStmtsDocList parseYaml(Logger logger, String data) {
|
||||
public RawOpsDocList parseYaml(String data) {
|
||||
LoadSettings loadSettings = LoadSettings.builder().build();
|
||||
Load yaml = new Load(loadSettings);
|
||||
Iterable<Object> objects = yaml.loadAllFromString(data);
|
||||
List<RawStmtsDoc> newDocList = new ArrayList<>();
|
||||
List<RawOpsDoc> newDocList = new ArrayList<>();
|
||||
|
||||
for (Object object : objects) {
|
||||
if (object instanceof Map) {
|
||||
RawStmtsDoc doc = new RawStmtsDoc();
|
||||
RawOpsDoc doc = new RawOpsDoc();
|
||||
Map<String, Object> docfields = (Map<String, Object>) object;
|
||||
doc.setFieldsByReflection(docfields);
|
||||
if (docfields.size()>0) {
|
||||
@ -96,24 +95,11 @@ public class RawStmtsLoader {
|
||||
|
||||
newDocList.add(doc);
|
||||
} else {
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
|
||||
}
|
||||
}
|
||||
RawStmtsDocList rawStmtsDocList = new RawStmtsDocList(newDocList);
|
||||
return rawStmtsDocList;
|
||||
}
|
||||
|
||||
protected String applyTransforms(Logger logger, String data) {
|
||||
for (Function<String, String> xform : stringTransformers) {
|
||||
try {
|
||||
if (logger != null) logger.trace(() -> "Applying string transformer to yaml data:" + xform);
|
||||
data = xform.apply(data);
|
||||
} catch (Exception e) {
|
||||
RuntimeException t = new OpConfigError("Error applying string transforms to input", e);
|
||||
throw t;
|
||||
}
|
||||
}
|
||||
return data;
|
||||
RawOpsDocList rawOpsDocList = new RawOpsDocList(newDocList);
|
||||
return rawOpsDocList;
|
||||
}
|
||||
|
||||
}
|
@ -1,48 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.rawyaml;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class RawStmtsDocList {
|
||||
|
||||
private final List<RawStmtsDoc> rawStmtsDocList;
|
||||
|
||||
public RawStmtsDocList(List<RawStmtsDoc> rawStmtsDocList) {
|
||||
this.rawStmtsDocList = rawStmtsDocList;
|
||||
}
|
||||
|
||||
public static RawStmtsDocList forSingleStatement(String statement) {
|
||||
RawStmtsDoc rawStmtsDoc = RawStmtsDoc.forSingleStatement(statement);
|
||||
return new RawStmtsDocList(List.of(rawStmtsDoc));
|
||||
}
|
||||
|
||||
public static RawStmtsDocList none() {
|
||||
return new RawStmtsDocList(List.of());
|
||||
}
|
||||
|
||||
public List<RawStmtsDoc> getStmtsDocs() {
|
||||
return rawStmtsDocList;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
int docs = rawStmtsDocList.size();
|
||||
int blocks = rawStmtsDocList.stream().map(RawStmtsDoc::getBlocks).mapToInt(List::size).sum();
|
||||
long optemplates = rawStmtsDocList.stream().flatMap(d -> d.getBlocks().stream()).flatMap(s -> s.getRawStmtDefs().stream()).count();
|
||||
return "docs:" + docs + " blocks:" + blocks + " optemplates:" + optemplates;
|
||||
}
|
||||
}
|
@ -76,7 +76,7 @@ public class RawYamlLoader {
|
||||
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
|
||||
Load yaml = new Load(LoadSettings.builder().build());
|
||||
Iterable<Object> objects = yaml.loadAllFromString(data);
|
||||
List<RawStmtsDoc> newDocList = new ArrayList<>();
|
||||
List<RawOpsDoc> newDocList = new ArrayList<>();
|
||||
|
||||
List<Map<String,Object>> maps = new ArrayList<>();
|
||||
|
||||
@ -84,7 +84,7 @@ public class RawYamlLoader {
|
||||
if (object instanceof Map) {
|
||||
maps.add(new LinkedHashMap<>((Map<String,Object>)object));
|
||||
} else {
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a statements yaml doc: " + object.getClass().getCanonicalName());
|
||||
throw new RuntimeException("Unable to coerce a non-map type to a workload structure: " + object.getClass().getCanonicalName());
|
||||
}
|
||||
}
|
||||
return maps;
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -17,7 +17,7 @@
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef;
|
||||
import io.nosqlbench.api.errors.BasicError;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -27,15 +27,15 @@ import java.util.*;
|
||||
public class OpDef extends OpTemplate {
|
||||
private final static Logger logger = LogManager.getLogger(OpDef.class);
|
||||
|
||||
private final RawStmtDef rawStmtDef;
|
||||
private final StmtsBlock block;
|
||||
private final RawOpDef rawOpDef;
|
||||
private final OpsBlock block;
|
||||
private final LinkedHashMap<String, Object> params;
|
||||
private final LinkedHashMap<String, String> bindings;
|
||||
private final LinkedHashMap<String, String> tags;
|
||||
|
||||
public OpDef(StmtsBlock block, RawStmtDef rawStmtDef) {
|
||||
public OpDef(OpsBlock block, RawOpDef rawOpDef) {
|
||||
this.block = block;
|
||||
this.rawStmtDef = rawStmtDef;
|
||||
this.rawOpDef = rawOpDef;
|
||||
this.params = composeParams();
|
||||
this.bindings = composeBindings();
|
||||
this.tags = composeTags();
|
||||
@ -43,12 +43,12 @@ public class OpDef extends OpTemplate {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return block.getName() + "--" + rawStmtDef.getName();
|
||||
return block.getName() + "--" + rawOpDef.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Map<String, Object>> getOp() {
|
||||
Object op = rawStmtDef.getOp();
|
||||
Object op = rawOpDef.getOp();
|
||||
if (op == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
@ -75,7 +75,7 @@ public class OpDef extends OpTemplate {
|
||||
}
|
||||
|
||||
private LinkedHashMap<String, String> composeBindings() {
|
||||
MultiMapLookup<String> lookup = new MultiMapLookup<>(rawStmtDef.getBindings(), block.getBindings());
|
||||
MultiMapLookup<String> lookup = new MultiMapLookup<>(rawOpDef.getBindings(), block.getBindings());
|
||||
return new LinkedHashMap<>(lookup);
|
||||
}
|
||||
|
||||
@ -85,7 +85,7 @@ public class OpDef extends OpTemplate {
|
||||
}
|
||||
|
||||
private LinkedHashMap<String, Object> composeParams() {
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtDef.getParams(), block.getParams());
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpDef.getParams(), block.getParams());
|
||||
LinkedHashMap<String, Object> params = new LinkedHashMap<>(lookup);
|
||||
return params;
|
||||
}
|
||||
@ -98,7 +98,7 @@ public class OpDef extends OpTemplate {
|
||||
}
|
||||
|
||||
private LinkedHashMap<String, String> composeTags() {
|
||||
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawStmtDef.getTags(), block.getTags()));
|
||||
LinkedHashMap<String, String> tagsWithName = new LinkedHashMap<>(new MultiMapLookup<>(rawOpDef.getTags(), block.getTags()));
|
||||
tagsWithName.put("name",getName());
|
||||
tagsWithName.put("block",block.getName());
|
||||
return tagsWithName;
|
||||
@ -111,7 +111,7 @@ public class OpDef extends OpTemplate {
|
||||
|
||||
@Override
|
||||
public String getDesc() {
|
||||
return rawStmtDef.getDesc();
|
||||
return rawOpDef.getDesc();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -32,118 +32,17 @@ import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* <p>The OpTemplate is the developer's view of the operational templates that users
|
||||
* provide in YAML or some other structured format.</p>
|
||||
*
|
||||
* <H2>Terms</H2>
|
||||
* Within this documentation, the word <i>OpTemplate</i> will refer to the template API and
|
||||
* semantics. The word <i>user template</i> will refer to the configuration data as provided
|
||||
* by a user.
|
||||
*
|
||||
* <p>OpTemplates are the native Java representation of the user templates that specify how to
|
||||
* make an executable operation. OpTemplates are not created for each operation, but are used
|
||||
* to create an mostly-baked intermediate form commonly known as a <i>ready op</i>.
|
||||
* It is the intermediate form which is used to create an instance of an executable
|
||||
* op in whichever way is the most appropriate and efficient for a given driver.</p>
|
||||
*
|
||||
* <p>This class serves as the canonical documentation and API for how user templates
|
||||
* are mapped into a fully resolved OpTemplate. User-provided op templates can be
|
||||
* any basic data structure, and are often provided as part of a YAML workload file.
|
||||
* The description below will focus on structural rules rather than any particular
|
||||
* encoding format. The types used are fairly universal and easy to map from one
|
||||
* format to another.</p>
|
||||
*
|
||||
*
|
||||
* <p>A long-form introduction to this format is included in the main NoSQLBench docs
|
||||
* at <a href="http://docs.nosqlbench.io">docs.nosqlbench.io</a>
|
||||
* under the <I>Designing Workloads</I> section.</p>
|
||||
*
|
||||
* <p>A few structural variations are allowed -- No specific form enforced. The reasons for this are:
|
||||
* 1) It is generally obvious what as user wants to do from a given layout. 2) Data structure
|
||||
* markup is generally frustrating and difficult to troubleshoot. 3) The conceptual domain of
|
||||
* NB op construction is well-defined enough to avoid ambiguity.</p>
|
||||
*
|
||||
* <H2>Type Conventions</H2>
|
||||
*
|
||||
* For the purposes of simple interoperability, the types used at this interface boundary should
|
||||
* be limited to common scalar types -- numbers and strings, and simple structures like maps and lists.
|
||||
* The basic types defined for ECMAScript should eventually be supported, but no domain-specific
|
||||
* objects which would require special encoding or decoding rules should be used.
|
||||
*
|
||||
* <H2>Standard Properties</H2>
|
||||
*
|
||||
* Each op template can have these standard properties:
|
||||
* <UL>
|
||||
* <LI>name - every op template has a name, even if it is auto generated for you. This is used to
|
||||
* name errors in the log, to name metrics in telemetry, and so on.</LI>
|
||||
* <LI>description - an optional description, defaulted to "".</LI>
|
||||
* <LI>statement - An optional string value which represents an opaque form of the body of
|
||||
* an op template</LI>
|
||||
* <LI>params - A string-object map of zero or more named parameters, where the key is taken as the parameter
|
||||
* name and the value is any simple object form as limited by type conventions above.
|
||||
* <LI>bindings - A map of binding definitions, where the string key is taken as the anchor name, and the
|
||||
* string value is taken as the binding recipe.</LI>
|
||||
* <LI>tags - A map of tags, with string names and values</LI>
|
||||
* </UL>
|
||||
*
|
||||
* The user-provided definition of an op template should capture a blueprint of an operation to be executed by
|
||||
* a native driver. As such, you need either a statement or a set of params which can describe what
|
||||
* specific type should be constructed. The rules on building an executable operation are not enforced
|
||||
* by this API. Yet, responsible NB driver developers will clearly document what the rules
|
||||
* are for specifying each specific type of operation supported by an NB driver with examples in YAML format.
|
||||
*
|
||||
* <H2>OpTemplate Construction Rules</H2>
|
||||
*
|
||||
* <p>The available structural forms follow a basic set of rules for constructing the OpTemplate in a consistent way.
|
||||
* <OL>
|
||||
* <LI>A collection of user-provided op templates is provided as a string, a list or a map.</LI>
|
||||
* <LI>All maps are order-preserving, like {@link java.util.LinkedHashMap}</LI>
|
||||
* <LI>For maps, the keys are taken as the names of the op template instances.</LI>
|
||||
* <LI>The content of each op template can be provided as a string or as a map.</LI>
|
||||
* <OL>
|
||||
* <LI>If the op template entry is provided as a string, then the OpTemplate is constructed as having only a single
|
||||
* <i>statement</i> property (in addition to defaults within scope).
|
||||
* as provided by OpTemplate API.</LI>
|
||||
* <LI>If the op template entry is provided as a map, then the OpTemplate is constructed as having all of the
|
||||
* named properties defined in the standard properties above.
|
||||
* Any entry in the template which is not a reserved word is assigned to the params map as a parameter, in whatever structured
|
||||
* type is appropriate (scalar, lists, maps).</LI>
|
||||
* </LI>
|
||||
* <p>
|
||||
* The OpTemplate is a structurally normalized type which presents the user-provided op template to the NoSQLBench
|
||||
* loading and templating mechanisms. This type is not generally used directly for new driver development. It is the
|
||||
* backing data which is used by {@link io.nosqlbench.engine.api.templating.ParsedOp}, which is used in drivers to map
|
||||
* op templates to function to be used for a given cycle value.
|
||||
* </p>
|
||||
* </OL>
|
||||
*
|
||||
* <H2>Example Forms</H2>
|
||||
* The valid forms are shown below as examples.
|
||||
*
|
||||
* <H3>One String Statement</H3>
|
||||
* <pre>{@code
|
||||
* statement: statement
|
||||
* }</pre>
|
||||
*
|
||||
* <H3>List of Templates</H3>
|
||||
* <pre>{@code
|
||||
* statements:
|
||||
* - statement1
|
||||
* - statement2
|
||||
* }</pre>
|
||||
*
|
||||
* <H3>List of Maps</H3>
|
||||
* <pre>{@code
|
||||
* statements:
|
||||
* - name: name1
|
||||
* stmt: statement body
|
||||
* params:
|
||||
* p1: v1
|
||||
* p2: v2
|
||||
* }</pre>
|
||||
*
|
||||
* <H3>List Of Condensed Maps</H3>
|
||||
* <pre>{@code
|
||||
* statements:
|
||||
* - name1: statement body
|
||||
* p1: v1
|
||||
* p2: v2
|
||||
* }</pre>
|
||||
* <p>
|
||||
* This is part of the implementation of the NoSQLBench <em>Uniform Workload Specification</em>. Check the tests
|
||||
* for UniformWorkloadSpecification directly to see how this specification is tested and documented.
|
||||
* </p>
|
||||
*/
|
||||
public abstract class OpTemplate implements Tagged {
|
||||
|
||||
@ -170,6 +69,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
/**
|
||||
* Return a map of tags for this statement. Implementations are required to
|
||||
* add a tag for "name" automatically when this value is set during construction.
|
||||
*
|
||||
* @return A map of assigned tags for the op, with the name added as an auto-tag.
|
||||
*/
|
||||
public abstract Map<String, String> getTags();
|
||||
@ -208,7 +108,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
if (defaultValue.getClass().isAssignableFrom(value.getClass())) {
|
||||
return (V) value;
|
||||
} else {
|
||||
return NBTypeConverter.convertOr(value,defaultValue);
|
||||
return NBTypeConverter.convertOr(value, defaultValue);
|
||||
}
|
||||
|
||||
}
|
||||
@ -229,7 +129,6 @@ public abstract class OpTemplate implements Tagged {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public <V> V getParam(String name, Class<? extends V> type) {
|
||||
Object object = getParams().get(name);
|
||||
if (object == null) {
|
||||
@ -273,7 +172,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
*
|
||||
* @return an optional {@link ParsedStringTemplate}
|
||||
*/
|
||||
public Optional<ParsedStringTemplate> getParsed(Function<String,String>... rewriters) {
|
||||
public Optional<ParsedStringTemplate> getParsed(Function<String, String>... rewriters) {
|
||||
Optional<String> os = getStmt();
|
||||
return os.map(s -> {
|
||||
String result = s;
|
||||
@ -281,7 +180,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
result = rewriter.apply(result);
|
||||
}
|
||||
return result;
|
||||
}).map(s -> new ParsedStringTemplate(s,getBindings()));
|
||||
}).map(s -> new ParsedStringTemplate(s, getBindings()));
|
||||
}
|
||||
|
||||
public Optional<ParsedStringTemplate> getParsed() {
|
||||
@ -309,7 +208,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
fields.put(FIELD_TAGS, this.getTags());
|
||||
}
|
||||
|
||||
this.getOp().ifPresent(o -> fields.put(FIELD_OP,o));
|
||||
this.getOp().ifPresent(o -> fields.put(FIELD_OP, o));
|
||||
|
||||
fields.put(FIELD_NAME, this.getName());
|
||||
|
||||
@ -320,10 +219,11 @@ public abstract class OpTemplate implements Tagged {
|
||||
* Legacy support for String form statements. This is left here as a convenience method,
|
||||
* however it is changed to an Optional to force caller refactorings.
|
||||
*
|
||||
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op fields at all.
|
||||
* @return An optional string version of the op, empty if there is no 'stmt' property in the op fields, or no op
|
||||
* fields at all.
|
||||
*/
|
||||
public Optional<String> getStmt() {
|
||||
return getOp().map(m->m.get("stmt")).map(s->{
|
||||
return getOp().map(m -> m.get("stmt")).map(s -> {
|
||||
if (s instanceof CharSequence) {
|
||||
return s.toString();
|
||||
} else {
|
||||
@ -333,7 +233,7 @@ public abstract class OpTemplate implements Tagged {
|
||||
}
|
||||
|
||||
public Element getParamReader() {
|
||||
return NBParams.one(getName(),getParams());
|
||||
return NBParams.one(getName(), getParams());
|
||||
}
|
||||
|
||||
/**
|
||||
@ -347,13 +247,13 @@ public abstract class OpTemplate implements Tagged {
|
||||
* @return the map of all remaining fields from the op template and the params map.
|
||||
*/
|
||||
public Map<String, Object> remainingFields() {
|
||||
Map<String,Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
|
||||
Map<String, Object> remaining = new LinkedHashMap<>(getOp().orElse(Map.of()));
|
||||
remaining.putAll(getParams());
|
||||
return remaining;
|
||||
}
|
||||
|
||||
public void assertConsumed() {
|
||||
if (size()>0) {
|
||||
if (size() > 0) {
|
||||
throw new OpConfigError("The op template named '" + getName() + "' was not fully consumed. These fields are not being applied:" + remainingFields());
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Copyright (c) 2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/**
|
||||
* The formats which are recognized as source data for the workload. Any serialization may be supported
|
||||
* which can be converted from a character string to an {@link io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList} structure.
|
||||
*
|
||||
* Those which are derived from paths may be determined by their filename extension. Others, which are provided from internal
|
||||
* NoSQLBench sources, may only be invoked explicitly.
|
||||
*/
|
||||
public enum OpTemplateFormat {
|
||||
/**
|
||||
* The source is in YAML format
|
||||
*/
|
||||
yaml("yaml", "yml"),
|
||||
|
||||
/**
|
||||
* The source is in JSON format
|
||||
*/
|
||||
json("json", "json5"),
|
||||
|
||||
/**
|
||||
* The source is in jsonnet format, suitable for pre-processing with context data
|
||||
*/
|
||||
jsonnet("jsonnet", "jsonnet5"),
|
||||
|
||||
/**
|
||||
* The source is inline, meaning {@link io.nosqlbench.api.config.params.ParamsParser} format.
|
||||
* This is similar to the {@link #stmt} format except that it is parsed for internal op structure.
|
||||
* This format is not detectable by source path, and thus can only be used when provided directly
|
||||
* from the caller.
|
||||
*/
|
||||
inline(),
|
||||
|
||||
/**
|
||||
* The source is in single-statement form, meaning that it is known to be the value of the 'stmt'
|
||||
* field of an op template. This format is not detectable by source path, and thus can only be used when provided directly
|
||||
* from the caller.
|
||||
*/
|
||||
stmt();
|
||||
|
||||
private final String[] pathExtensions;
|
||||
|
||||
OpTemplateFormat(String... pathExtensions) {
|
||||
this.pathExtensions = pathExtensions;
|
||||
}
|
||||
|
||||
public static OpTemplateFormat valueOfURI(URI uri) {
|
||||
var fullName = uri.toString();
|
||||
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
|
||||
|
||||
for (OpTemplateFormat value : values()) {
|
||||
for (String pathExtension : value.pathExtensions) {
|
||||
if (pathExtension.equals(extension)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new RuntimeException("Unable to determine source format for " + uri);
|
||||
|
||||
}
|
||||
|
||||
public static OpTemplateFormat valueOfPath(Path path) {
|
||||
var fullName = path.toString();
|
||||
String extension = fullName.substring(fullName.lastIndexOf('.')+1).toLowerCase();
|
||||
|
||||
for (OpTemplateFormat value : values()) {
|
||||
for (String pathExtension : value.pathExtensions) {
|
||||
if (pathExtension.equals(extension)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new RuntimeException("Unable to determine source format for " + path);
|
||||
}
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -17,34 +17,34 @@
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.MultiMapLookup;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtDef;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpDef;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock;
|
||||
import io.nosqlbench.api.engine.util.Tagged;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
public class OpsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
|
||||
private final RawStmtsBlock rawStmtsBlock;
|
||||
private final StmtsDoc rawStmtsDoc;
|
||||
private final RawOpsBlock rawOpsBlock;
|
||||
private final OpsDoc rawOpsDoc;
|
||||
private final int blockIdx;
|
||||
|
||||
|
||||
public StmtsBlock(RawStmtsBlock rawStmtsBlock, StmtsDoc rawStmtsDoc, int blockIdx) {
|
||||
this.rawStmtsBlock = rawStmtsBlock;
|
||||
this.rawStmtsDoc = rawStmtsDoc;
|
||||
public OpsBlock(RawOpsBlock rawOpsBlock, OpsDoc rawOpsDoc, int blockIdx) {
|
||||
this.rawOpsBlock = rawOpsBlock;
|
||||
this.rawOpsDoc = rawOpsDoc;
|
||||
this.blockIdx = blockIdx;
|
||||
}
|
||||
|
||||
public List<OpTemplate> getOps() {
|
||||
|
||||
List<OpTemplate> rawOpTemplates = new ArrayList<>();
|
||||
List<RawStmtDef> statements = rawStmtsBlock.getRawStmtDefs();
|
||||
List<RawOpDef> opDefs = rawOpsBlock.getRawOpDefs();
|
||||
|
||||
for (int i = 0; i < statements.size(); i++) {
|
||||
for (int i = 0; i < opDefs.size(); i++) {
|
||||
rawOpTemplates.add(
|
||||
new OpDef(this, statements.get(i))
|
||||
new OpDef(this, opDefs.get(i))
|
||||
);
|
||||
}
|
||||
return rawOpTemplates;
|
||||
@ -52,11 +52,11 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
|
||||
public String getName() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
if (!rawStmtsDoc.getName().isEmpty()) {
|
||||
sb.append(rawStmtsDoc.getName()).append("--");
|
||||
if (!rawOpsDoc.getName().isEmpty()) {
|
||||
sb.append(rawOpsDoc.getName()).append("--");
|
||||
}
|
||||
if (!rawStmtsBlock.getName().isEmpty()) {
|
||||
sb.append(rawStmtsBlock.getName());
|
||||
if (!rawOpsBlock.getName().isEmpty()) {
|
||||
sb.append(rawOpsBlock.getName());
|
||||
} else {
|
||||
sb.append("block").append(blockIdx);
|
||||
}
|
||||
@ -64,15 +64,15 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
}
|
||||
|
||||
public Map<String, String> getTags() {
|
||||
return new MultiMapLookup<>(rawStmtsBlock.getTags(), rawStmtsDoc.getTags());
|
||||
return new MultiMapLookup<>(rawOpsBlock.getTags(), rawOpsDoc.getTags());
|
||||
}
|
||||
|
||||
public Map<String, Object> getParams() {
|
||||
return new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
|
||||
return new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
|
||||
}
|
||||
|
||||
public Map<String, String> getParamsAsText() {
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
|
||||
LinkedHashMap<String, String> stringmap = new LinkedHashMap<>();
|
||||
lookup.forEach((k, v) -> stringmap.put(k, v.toString()));
|
||||
return stringmap;
|
||||
@ -81,7 +81,7 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
@SuppressWarnings("unchecked")
|
||||
public <V> V getParamOrDefault(String name, V defaultValue) {
|
||||
Objects.requireNonNull(defaultValue);
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
|
||||
if (!lookup.containsKey(name)) {
|
||||
return defaultValue;
|
||||
}
|
||||
@ -90,14 +90,14 @@ public class StmtsBlock implements Tagged, Iterable<OpTemplate> {
|
||||
}
|
||||
|
||||
public <V> V getParam(String name, Class<? extends V> type) {
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawStmtsBlock.getParams(), rawStmtsDoc.getParams());
|
||||
MultiMapLookup<Object> lookup = new MultiMapLookup<>(rawOpsBlock.getParams(), rawOpsDoc.getParams());
|
||||
Object object = lookup.get(name);
|
||||
V value = type.cast(object);
|
||||
return value;
|
||||
}
|
||||
|
||||
public Map<String, String> getBindings() {
|
||||
return new MultiMapLookup<>(rawStmtsBlock.getBindings(), rawStmtsDoc.getBindings());
|
||||
return new MultiMapLookup<>(rawOpsBlock.getBindings(), rawOpsDoc.getBindings());
|
||||
}
|
||||
|
||||
@Override
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,8 +16,8 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsBlock;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsBlock;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc;
|
||||
import io.nosqlbench.api.engine.util.Tagged;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@ -27,29 +27,29 @@ import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* StmtsDoc creates a logical view of a statements doc that includes
|
||||
* OpsDoc creates a logical view of a RawOpsDoc doc that includes
|
||||
* all inherited and overridden values for bindings, tags, and params.
|
||||
*/
|
||||
public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
|
||||
public class OpsDoc implements Tagged, Iterable<OpsBlock> {
|
||||
|
||||
private final RawStmtsDoc rawStmtsDoc;
|
||||
private final RawOpsDoc rawOpsDoc;
|
||||
|
||||
public StmtsDoc(RawStmtsDoc rawStmtsDoc) {
|
||||
this.rawStmtsDoc = rawStmtsDoc;
|
||||
public OpsDoc(RawOpsDoc rawOpsDoc) {
|
||||
this.rawOpsDoc = rawOpsDoc;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a usable list of blocks, including inherited bindings, params, and tags
|
||||
* from the parent doc
|
||||
*/
|
||||
public List<StmtsBlock> getBlocks() {
|
||||
List<StmtsBlock> blocks = new ArrayList<>();
|
||||
public List<OpsBlock> getBlocks() {
|
||||
List<OpsBlock> blocks = new ArrayList<>();
|
||||
|
||||
int blockIdx = 0;
|
||||
for (RawStmtsBlock rawStmtsBlock : rawStmtsDoc.getBlocks()) {
|
||||
String compositeName = rawStmtsDoc.getName() +
|
||||
(rawStmtsBlock.getName().isEmpty() ? "" : "-" + rawStmtsBlock.getName());
|
||||
StmtsBlock compositeBlock = new StmtsBlock(rawStmtsBlock, this, ++blockIdx);
|
||||
for (RawOpsBlock rawOpsBlock : rawOpsDoc.getBlocks()) {
|
||||
String compositeName = rawOpsDoc.getName() +
|
||||
(rawOpsBlock.getName().isEmpty() ? "" : "-" + rawOpsBlock.getName());
|
||||
OpsBlock compositeBlock = new OpsBlock(rawOpsBlock, this, ++blockIdx);
|
||||
blocks.add(compositeBlock);
|
||||
}
|
||||
|
||||
@ -61,53 +61,53 @@ public class StmtsDoc implements Tagged, Iterable<StmtsBlock> {
|
||||
*/
|
||||
@Override
|
||||
public Map<String, String> getTags() {
|
||||
return rawStmtsDoc.getTags();
|
||||
return rawOpsDoc.getTags();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a usable map of parameters, including those inherited from the parent doc
|
||||
*/
|
||||
public Map<String, Object> getParams() {
|
||||
return rawStmtsDoc.getParams();
|
||||
return rawOpsDoc.getParams();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a usable map of bindings, including those inherited from the parent doc
|
||||
*/
|
||||
public Map<String, String> getBindings() {
|
||||
return rawStmtsDoc.getBindings();
|
||||
return rawOpsDoc.getBindings();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the name of this block
|
||||
*/
|
||||
public String getName() {
|
||||
return rawStmtsDoc.getName();
|
||||
return rawOpsDoc.getName();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The list of all included statements for all included block in this document,
|
||||
* including the inherited and overridden values from the this doc and the parent block.
|
||||
* @return The list of all included op templates for all included block in this document,
|
||||
* including the inherited and overridden values from this doc and the parent block.
|
||||
*/
|
||||
public List<OpTemplate> getStmts() {
|
||||
public List<OpTemplate> getOpTemplates() {
|
||||
return getBlocks().stream().flatMap(b -> b.getOps().stream()).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Allow StmtsDoc to be used in iterable loops.
|
||||
* @return An iterator of {@link StmtsBlock}
|
||||
* @return An iterator of {@link OpsBlock}
|
||||
*/
|
||||
@Override
|
||||
public Iterator<StmtsBlock> iterator() {
|
||||
public Iterator<OpsBlock> iterator() {
|
||||
return getBlocks().iterator();
|
||||
}
|
||||
|
||||
|
||||
public Scenarios getScenarios() {
|
||||
return new Scenarios(rawStmtsDoc.getRawScenarios());
|
||||
return new Scenarios(rawOpsDoc.getRawScenarios());
|
||||
}
|
||||
|
||||
public String getDescription() {
|
||||
return rawStmtsDoc.getDesc();
|
||||
return rawOpsDoc.getDesc();
|
||||
}
|
||||
}
|
@ -16,9 +16,9 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsDocList;
|
||||
import io.nosqlbench.engine.api.util.TagFilter;
|
||||
import io.nosqlbench.api.config.standard.ConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfigModel;
|
||||
@ -31,48 +31,48 @@ import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class StmtsDocList implements Iterable<StmtsDoc> {
|
||||
private final static Logger logger = LogManager.getLogger(StmtsDocList.class);
|
||||
public class OpsDocList implements Iterable<OpsDoc> {
|
||||
private final static Logger logger = LogManager.getLogger(OpsDocList.class);
|
||||
|
||||
private final RawStmtsDocList rawStmtsDocList;
|
||||
private final RawOpsDocList rawOpsDocList;
|
||||
private final Map<String, String> templateVariables = new LinkedHashMap<>();
|
||||
|
||||
public StmtsDocList(RawStmtsDocList rawStmtsDocList) {
|
||||
this.rawStmtsDocList = rawStmtsDocList;
|
||||
public OpsDocList(RawOpsDocList rawOpsDocList) {
|
||||
this.rawOpsDocList = rawOpsDocList;
|
||||
}
|
||||
|
||||
public static StmtsDocList none() {
|
||||
return new StmtsDocList(RawStmtsDocList.none());
|
||||
public static OpsDocList none() {
|
||||
return new OpsDocList(RawOpsDocList.none());
|
||||
}
|
||||
|
||||
public List<StmtsDoc> getStmtDocs(String tagFilter) {
|
||||
public List<OpsDoc> getStmtDocs(String tagFilter) {
|
||||
TagFilter tf = new TagFilter(tagFilter);
|
||||
return getStmtDocs().stream()
|
||||
.filter(tf::matchesTagged)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<StmtsDoc> getStmtDocs() {
|
||||
return rawStmtsDocList.getStmtsDocs().stream()
|
||||
.map(StmtsDoc::new)
|
||||
public List<OpsDoc> getStmtDocs() {
|
||||
return rawOpsDocList.getOpsDocs().stream()
|
||||
.map(OpsDoc::new)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<OpTemplate> getStmts() {
|
||||
return getStmts("");
|
||||
public List<OpTemplate> getOps() {
|
||||
return getOps("");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param tagFilterSpec a comma-separated tag filter spec
|
||||
* @return The list of all included statements for all included blocks of in this document,
|
||||
* including the inherited and overridden values from the this doc and the parent block.
|
||||
* @return The list of all included op templates for all included blocks of in this document,
|
||||
* including the inherited and overridden values from this doc and the parent block.
|
||||
*/
|
||||
public List<OpTemplate> getStmts(String tagFilterSpec) {
|
||||
public List<OpTemplate> getOps(String tagFilterSpec) {
|
||||
TagFilter ts = new TagFilter(tagFilterSpec);
|
||||
List<OpTemplate> opTemplates = new ArrayList<>();
|
||||
|
||||
getStmtDocs().stream()
|
||||
.flatMap(d -> d.getStmts().stream())
|
||||
.flatMap(d -> d.getOpTemplates().stream())
|
||||
.filter(ts::matchesTagged)
|
||||
.forEach(opTemplates::add);
|
||||
|
||||
@ -81,7 +81,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
|
||||
|
||||
|
||||
@Override
|
||||
public Iterator<StmtsDoc> iterator() {
|
||||
public Iterator<OpsDoc> iterator() {
|
||||
return getStmtDocs().iterator();
|
||||
}
|
||||
|
||||
@ -94,7 +94,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
|
||||
public Map<String, String> getDocBindings() {
|
||||
LinkedHashMap<String, String> docBindings = new LinkedHashMap<>();
|
||||
getStmtDocs().stream()
|
||||
.map(StmtsDoc::getBindings)
|
||||
.map(OpsDoc::getBindings)
|
||||
.forEach(docBindings::putAll);
|
||||
return docBindings;
|
||||
}
|
||||
@ -130,7 +130,7 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
|
||||
}
|
||||
|
||||
public NBConfigModel getConfigModel() {
|
||||
ConfigModel cfgmodel = ConfigModel.of(StmtsDocList.class);
|
||||
ConfigModel cfgmodel = ConfigModel.of(OpsDocList.class);
|
||||
getTemplateVariables().forEach((k, v) -> {
|
||||
cfgmodel.add(Param.defaultTo(k, v, "template parameter found in the yaml workload"));
|
||||
});
|
||||
@ -144,9 +144,9 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
|
||||
int blockscount = 0;
|
||||
int opscount = 0;
|
||||
|
||||
for (StmtsDoc stmtDoc : this.getStmtDocs()) {
|
||||
for (OpsDoc stmtDoc : this.getStmtDocs()) {
|
||||
docscount++;
|
||||
for (StmtsBlock block : stmtDoc.getBlocks()) {
|
||||
for (OpsBlock block : stmtDoc.getBlocks()) {
|
||||
blockscount++;
|
||||
for (OpTemplate op : block.getOps()) {
|
||||
opscount++;
|
||||
@ -155,18 +155,16 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
|
||||
}
|
||||
|
||||
sb.append("docs: " + docscount + " blocks:" + blockscount + " ops:" + opscount);
|
||||
// String names = this.rawStmtsDocList.getStmtsDocs().stream().flatMap(sd -> sd.getRawStmtDefs().stream()).map(d->d.getName()).collect(Collectors.joining(","));
|
||||
// sb.append(", names:").append(names);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static NBConfigModelExpander TEMPLATE_VAR_EXPANDER = workload -> {
|
||||
StmtsDocList loaded = StatementsLoader.loadPath(logger, (String) workload, "activities");
|
||||
OpsDocList loaded = OpsLoader.loadPath((String) workload, Map.of(), "activities");
|
||||
return loaded.getConfigModel();
|
||||
};
|
||||
|
||||
public Pattern getVersionRegex() {
|
||||
List<RawStmtsDoc> stmtDocs = rawStmtsDocList.getStmtsDocs();
|
||||
List<RawOpsDoc> stmtDocs = rawOpsDocList.getOpsDocs();
|
||||
return Pattern.compile(stmtDocs.size()>0 ? stmtDocs.get(0).getVersionRegex() : ".*");
|
||||
}
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -31,34 +31,24 @@ import java.util.function.Function;
|
||||
* a client API for advanced testing. The default instance should simply be named
|
||||
* {@code default}</p>
|
||||
*
|
||||
* <p>Most native drivers use some combination of fluent, functional, and declarative
|
||||
* patterns. These usually require you to keep access to a set of core
|
||||
* state-holding objects in order to construct new elements to drive operations with.
|
||||
* An example of this would be creating a statement from a session. It is necessary
|
||||
* to keep the session around in for when you create new statements. Maintaining
|
||||
* the session object is considered an essential part of idiomatic and efficient
|
||||
* use of the API. Further, you may have builders or factories that are created
|
||||
* from the session which should be cached as well. Keeping all these objects
|
||||
* together requires attaching them to a cohesive owning object -- That is the space
|
||||
* cache.</p>
|
||||
* <p>Most native drivers use some combination of fluent, functional, and declarative patterns. These usually require
|
||||
* you to keep access to a set of core state-holding objects in order to construct new elements to drive operations
|
||||
* with. An example of this would be creating an executable operation from a session object. It is necessary to keep the
|
||||
* session around in for when you create new statements. Maintaining the session object is considered an essential part
|
||||
* of idiomatic and efficient use of the API. Further, you may have builders or factories that are created from the
|
||||
* session which should be cached as well. Keeping all these objects together requires attaching them to a cohesive
|
||||
* owning object -- That is the space cache.</p>
|
||||
*
|
||||
* <p>You might want to create multiple session contexts in order to test out
|
||||
* non-trivial behavior in advanced testing scenarios. To do this dynamically, i.e.
|
||||
* acquire some named space cache, simply call the {@link #get(String)}</p> method
|
||||
* with the name of the space you want to use. This value can be provided as a
|
||||
* dynamic field in your op mapping ({@link io.nosqlbench.engine.api.activityimpl.OpMapper})
|
||||
* or synthesis ({@link io.nosqlbench.engine.api.activityimpl.OpDispenser})
|
||||
* implementation.
|
||||
*
|
||||
* @param <S> The type which will represent the cache for a given type of adapter.
|
||||
* @param <S>
|
||||
* The type which will represent the cache for a given type of adapter.
|
||||
*/
|
||||
public class DriverSpaceCache<S> {
|
||||
|
||||
private final ConcurrentHashMap<String,S> cache = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, S> cache = new ConcurrentHashMap<>();
|
||||
|
||||
private final Function<String, S> newSpaceFunction;
|
||||
|
||||
public DriverSpaceCache(Function<String,S> newSpaceFunction) {
|
||||
public DriverSpaceCache(Function<String, S> newSpaceFunction) {
|
||||
this.newSpaceFunction = newSpaceFunction;
|
||||
}
|
||||
|
||||
@ -66,7 +56,7 @@ public class DriverSpaceCache<S> {
|
||||
return cache.computeIfAbsent(name, newSpaceFunction);
|
||||
}
|
||||
|
||||
public Map<String,S> getElements() {
|
||||
public Map<String, S> getElements() {
|
||||
return Collections.unmodifiableMap(cache);
|
||||
}
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -17,29 +17,34 @@
|
||||
package io.nosqlbench.engine.api.activityimpl.uniform.decorators;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.templating.DriverAdapterDecorators;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* In the case that no statements are provided for an activity, but a workload
|
||||
* description is provided, and the result for no found statements is not
|
||||
* simply a matter of tag filtering, an activity may provide its own
|
||||
* synthetic ops. This is here primarily to support the classic behavior of stdout
|
||||
* until it's functionality is subsumed by standard diagnostic features.
|
||||
*
|
||||
* Note that this is only valid while an activity uses a single driver, which will
|
||||
* change with upcoming API updates.
|
||||
* It is possible for a DriverAdapter to create op templates using partial information.
|
||||
* For example, the stdout driver can use only bindings to create CSV or JSON style data renderings.
|
||||
* This mechanism is only triggered when:
|
||||
* <OL>
|
||||
* <LI>No op templates were provided.</LI>
|
||||
* <LI>The default driver for an activity implements this method.</LI>
|
||||
* </OL>
|
||||
* <p>
|
||||
* This excludes cases where a workload was provided with op templates, but they were all filtered out. In that case,
|
||||
* the user should be informed of an error.
|
||||
*/
|
||||
public interface SyntheticOpTemplateProvider extends DriverAdapterDecorators {
|
||||
|
||||
/**
|
||||
* If a driver adapter supports creating example op templates from bindings,
|
||||
* it must implement this method to do so.
|
||||
* @param stmtsDocList The existing doc structure, which should contain no fully defined op templates, but may contain other elements like bindings
|
||||
*
|
||||
* @param opsDocList
|
||||
* The existing doc structure, which should contain no fully defined op templates, but may contain other
|
||||
* elements like bindings
|
||||
* @return A list of op templates, size zero or more
|
||||
*/
|
||||
List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String,Object> params);
|
||||
List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> params);
|
||||
}
|
||||
|
@ -1,34 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.templating;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* An Op Template Supplier can provide its own source of op templates instead
|
||||
* of relying on the built-in mechanism. By default, the built-in mechanism
|
||||
* will read op definitions from parameters first, then any ops (statements)
|
||||
* from yaml files provided in the workload= activity parameters.
|
||||
*/
|
||||
public interface OpTemplateSupplier extends DriverAdapterDecorators {
|
||||
|
||||
Optional<List<OpTemplate>> loadOpTemplates(NBConfiguration cfg);
|
||||
}
|
@ -0,0 +1,42 @@
|
||||
# Workload Specification
|
||||
|
||||
This directory contains the testable specification for workload definitions used by NoSQLBench.
|
||||
All the content blocks in this section have been validated with the latest NoSQLBench build.
|
||||
|
||||
Usually, users will not need to delve too deeply into this section. It is useful as a detailed
|
||||
guide for contributors and driver developers. If you are using a driver which leaves you
|
||||
wondering what a good op template example looks like, then the driver needs better examples in
|
||||
its documentation!
|
||||
|
||||
# Synopsis
|
||||
|
||||
There are two primary views of workload definitions that we care about:
|
||||
|
||||
1. The User View of **op templates**
|
||||
1. Op templates are simply the schematic recipes for building an operation once you know the
|
||||
cycle it is for.
|
||||
2. Op templates are provided by users in YAML or JSON or even directly via runtime API. This
|
||||
is called a workload template, which contains op templates.
|
||||
3. Op templates can be provided with optional metadata which serve to label, group,
|
||||
parameterize or otherwise make the individual op templates more manageable.
|
||||
4. A variety of forms are supported which are self-evident, but which allow users to have
|
||||
some flexibility in how they structure their YAML, JSON, or runtime collections. **This
|
||||
specification is about how these various forms are allowed, and how they relate to a
|
||||
fully-qualified and de-normalized op template view.
|
||||
2. The Developer View of the ParsedOp API. This is the view of an op template which presents the
|
||||
developer with a very high-level toolkit for building op synthesis functions.
|
||||
|
||||
# Details
|
||||
|
||||
The documentation in this directory serve as a testable specification for all the above. It
|
||||
shows specific examples of all the valid op template forms in both YAML and JSON, as well as how
|
||||
the data is normalized to feed developer's view of the ParsedOp API.
|
||||
|
||||
## Related Reading
|
||||
|
||||
If you want to understand the rest of this document, it is crucial that you have a working knowledge
|
||||
of the standard YAML format and several examples from the current drivers. You can learn this from
|
||||
the main documentation which demonstrates step-by-step how to build a workload. Reading further in
|
||||
this document will be most useful for core NB developers, or advanced users who want to know all
|
||||
the possible ways of building workloads.
|
||||
|
@ -0,0 +1,205 @@
|
||||
# ParsedOp API
|
||||
|
||||
In the workload template examples, we show statements as being formed from a string value. This is a
|
||||
specific type of statement form, although it is possible to provide structured op templates as well.
|
||||
|
||||
**The ParsedOp API is responsible for converting all valid op template forms into a consistent and
|
||||
unambiguous model.** Thus, the rules for mapping the various forms to the command model must be
|
||||
precise. Those rules are the substance of this specification.
|
||||
|
||||
## Op Synthesis
|
||||
|
||||
Executable operations are _created_ on the fly by NoSQLBench via a process called _Op Synthesis_.
|
||||
This is done incrementally in stages. The following narrative describes this process in logical
|
||||
stages. (The implementation may vary from this, but it explains the effects, nonetheless.)
|
||||
|
||||
Everything here happens **during** activity initialization, before the activity starts running
|
||||
cycles:
|
||||
|
||||
1. *Template Variable Expansion* - If there are template variables, such as
|
||||
`TEMPLATE(name,defaultval)` or `<<name:defaultval>>`, then these are expanded
|
||||
according to their defaults and any overrides provided in the activity params. This is a macro
|
||||
substitution only, so the values are simply interposed into the character stream of the document.
|
||||
2. *Jsonnet Evaluation* - If the source file was in jsonnet format (the extension was `.jsonnet`)
|
||||
then it is interpreted by sjsonnet, with all activity parameters available as external variables.
|
||||
3. *Structural Normalization* - The workload template (yaml, json, or data structure) is loaded
|
||||
into memory and transformed into a standard format. This means taking various list and map
|
||||
forms at every level and converting them to a singular standard form in memory.
|
||||
4. *Auto-Naming* - All elements which do not already have a name are assigned a simple name like
|
||||
`block2` or `op3`.
|
||||
5. *Auto-Tagging* - All op templates are given standard tag values under reserved tag names:
|
||||
- **block**: the name of the block containing the op template. For example: `block2`.
|
||||
- **name**: the name of the op template, prefixed with the block value and `--`. For example,
|
||||
`block2--op1`.
|
||||
6. *Property De-normalization* - Default values for all the standard op template properties are
|
||||
copied from the doc to the block layer unless the same-named key exists. Then the same
|
||||
method is applied from the doc layer to the op template layer. **At this point, the op
|
||||
templates are effectively an ordered list of data structures, each containing all necessary
|
||||
details for use.**
|
||||
7. *Tag Filtering* - The activity's `tag` param is used to filter all the op templates
|
||||
according to their tag map.
|
||||
8. *Bind Point and Capture Points* - Each op template is now converted into a ParsedOp, which is
|
||||
a swiss-army knife of op template introspection and function generation. It is the direct
|
||||
programmatic API that driver adapters use in subsequent steps.
|
||||
- Any string sequences with bind points like `this has a {bindpoint}` are automatically
|
||||
converted to a long -> string function.
|
||||
- Any direct references with no surrounding text like `{bindpoint}` are automatically
|
||||
converted to direct binding references.
|
||||
- Any other string form is cached as a static value.
|
||||
- The same process is applied to Lists and Maps, allowing structural templates which read
|
||||
like JSON with bind points in arbitrary places.
|
||||
8. *Op Mapping* - Using the ParsedOp API, each op template is categorized by the active `driver`
|
||||
according to that driver's documented examples and type-matching rules. Once the op mapper
|
||||
determines what op type a user intended, it uses this information and the associated op
|
||||
fields to create an *Op Dispenser*.
|
||||
9. *Op Sequencing* - The op dispensers are kept as an internal sequence, and installed into a
|
||||
[LUT](https://en.wikipedia.org/wiki/Lookup_table) according to their ratios and the specified
|
||||
(or default) sequencer. By default, round-robin with bucket exhaustion is used. The ratios
|
||||
specified are used directly in the LUT.
|
||||
|
||||
When this is complete, you are left with an efficient lookup table which indexes into a set of
|
||||
OpDispensers. The length of this lookup table is called the _sequence length_, and that value is
|
||||
used, by default, to set the _stride_ for the activity. This stride determines the size of
|
||||
per-thread cycle batching, effectively turning each sequence into a thread-safe set of
|
||||
operations which are serialized, and thus suitable for testing linearized operations with
|
||||
suitable dependency and error-handling mechanisms. (But wait, there's more!)
|
||||
|
||||
## Special Cases
|
||||
|
||||
Drivers are assigned to op templates individually, meaning you can specify the driver within an
|
||||
op template, not even assigning a default for the activity. Further, certain drivers are able to
|
||||
fill in missing details for op templates, like the `stdout` driver which only requires bindings.
|
||||
|
||||
This means that there are distinct cases for configuration which are valid, and these are
|
||||
checked at initialization time:
|
||||
|
||||
- A `driver` must be selected for each op template either directly or via activity params.
|
||||
- If the whole workload template provided does not include actual op templates **AND** a
|
||||
default driver is provided which can create synthetic op templates, it is given the raw
|
||||
workload template, incomplete as it is, and asked to provide op templates which have all
|
||||
normalization, naming, etc. already done. This is injected before the tag-filtering phase.
|
||||
- In any case that an actual non-zero list of op templates is provided and tag filtering removes
|
||||
them all, an error is thrown.
|
||||
- If, after tag filtering no op template are in the active list, an error is thrown.
|
||||
|
||||
# The ParsedOp
|
||||
|
||||
The components of a fully-parsed op template (AKA a ParsedOp) are:
|
||||
|
||||
## name
|
||||
|
||||
Each ParsedOp knows its name, which is simply the op template name that it was made from. This
|
||||
is useful for diagnostics, logging, and metrics.
|
||||
|
||||
## description
|
||||
|
||||
Every named element of a workload may be given a description.
|
||||
|
||||
## tags
|
||||
|
||||
Every op template has tags, even if they are auto-assigned from the block and op template names.
|
||||
If you assign explicit tags to an op template, the standard tags are still provided. Thus, it is
|
||||
an error to directly provide a tag named `block` or `name`.
|
||||
|
||||
## bindings
|
||||
|
||||
Although bindings are usually defined as workload template level property, they can also be
|
||||
provided directly as an op field property.
|
||||
|
||||
## op fields
|
||||
|
||||
The **op** property of an op template or ParsedOp is the root of the op fields. This is a map of
|
||||
specific fields specified by the user.
|
||||
|
||||
### static op fields
|
||||
|
||||
Some op fields are simply static values. Since these values are not generated per cycle, they are
|
||||
kept separate as reference data. Knowing which fields are static and which are not makes it
|
||||
possible for developers to optimize op synthesis.
|
||||
|
||||
### dynamic op fields
|
||||
|
||||
Other fields may be specified as recipes, with the actual value to be filled-in once the cycle
|
||||
value is known. All such fields are known as _dynamic op fields_, and are provided to the op
|
||||
dispenser as a long function, where the input is always the cycle value and the output is a
|
||||
type-specific value as determined by the associated binding recipe.
|
||||
|
||||
### bind points
|
||||
|
||||
This is how dynamic values are indicated. Each bind point in an op template results in some type of
|
||||
procedural generation binding. These can be references to named bindings elsewhere in the
|
||||
workload template, or they can be inline.
|
||||
|
||||
### capture points
|
||||
|
||||
Names of result values to save, and the variable names they are to be saved as. The names represent
|
||||
the name as it would be found in the native driver's API, such as the name `userid`
|
||||
in `select userid from ...`. In string form statements, users can specify that the userid should be
|
||||
saved as the thread-local variable named *userid* simply by tagging it
|
||||
like `select [userid] from ...`. They can also specify that this value should be captured under a
|
||||
different name with a variation like `select [userid as user_id] from ...`. This is the standard
|
||||
variable capture syntax for any string-based statement form.
|
||||
|
||||
### params
|
||||
|
||||
A backwards-compatible feature called op params is still available. This is another root
|
||||
property within an op template which can be used to accessorize op fields. By default, any op
|
||||
field which is not explicitly rooted under the `op` property are put there anyway. This is also
|
||||
true when there is an explicitly `params` property. However if the op property is provided, then
|
||||
all non-reserved fields are given to the params property instead. If both the `op` and the
|
||||
`param` op properties are specified, then no non-reserved op fields are allowed outside of these
|
||||
root values. Thus it is possible to still support params, but it is **highly** recommended that
|
||||
new driver developers avoid using this field, and instead allow all fields to be automatically
|
||||
anchored under the `op` property. This keeps configs terse and simple going forward.
|
||||
|
||||
Params may not be dynamic.
|
||||
|
||||
# Mapping Rules
|
||||
|
||||
A ParsedOp does not necessarily describe a specific low-level operation to be performed by
|
||||
a native driver. It *should* do so, but it is up to the user to provide a valid op template
|
||||
according to the documented rules of op construction for that driver type. These rules should be
|
||||
clearly documented by the driver developer as examples in markdown that is required for every
|
||||
driver. With this documentation, users can use `nb5 help <driver>` to see exactly how
|
||||
to create op templates for a given driver.
|
||||
|
||||
## String Form
|
||||
|
||||
Basic operations are made from a statement in some type of query language:
|
||||
|
||||
```yaml
|
||||
ops:
|
||||
- stringform: select [userid] from db.users where user='{username}';
|
||||
bindings:
|
||||
username: NumberNameToString()
|
||||
```
|
||||
|
||||
# Reserved op fields
|
||||
|
||||
The property names `ratio`, `driver`, `space`, are considered reserved by the NoSQLBench runtime.
|
||||
These are extracted and handled specially by the core runtime.
|
||||
|
||||
# Base OpDispenser fields
|
||||
|
||||
The BaseOpDispenser, which <s>is</s> will be required as the base implementation of any op
|
||||
dispenser going forward, provides cross-cutting functionality. These include `start-timers`,
|
||||
`stop-timers`, `instrument`, and likely will include more as future cross-driver functionality is
|
||||
added. These fields will be considered reserved property names.
|
||||
|
||||
# Optimization
|
||||
|
||||
It should be noted that the op mapping process, where user intentions are mapped from op templates to
|
||||
op dispensers is not something that needs to be done quickly. This occurs at _initialization_
|
||||
time. Instead, it is more important to focus on user experience factors, such as flexibility,
|
||||
obviousness, robustness, correctness, and so on. Thus, priority of design factors in this part
|
||||
of NB is placed more on clear and purposeful abstractions and less on optimizing for speed. The
|
||||
clarity and detail which is conveyed by this layer to the driver developer will then enable
|
||||
them to focus on building fast and correct op dispensers. These dispensers are also constructed
|
||||
before the workload starts running, but are used at high speed while the workload is running.
|
||||
|
||||
In essence:
|
||||
- Any initialization code which happens before or in the OpDispenser constructor should not be
|
||||
concerned with careful performance optimization.
|
||||
- Any code which occurs within the OpDispenser#apply method should be as lightweight as is
|
||||
reasonable.
|
||||
|
@ -1,99 +0,0 @@
|
||||
# Workload Specification
|
||||
|
||||
This directory contains the testable specification for workload definitions used by NoSQLBench.
|
||||
|
||||
## Op Templates vs Developer API
|
||||
There are two primary views of workload definitions that we care about:
|
||||
|
||||
1. The User View of **op templates**
|
||||
1. Op templates are simply the schematic recipes for building an operation.
|
||||
2. Op templates are provided by users in YAML or JSON or even directly via runtime API.
|
||||
3. Op templates can be provided with optional metadata which serves to label, group or
|
||||
otherwise make the individual op templates more manageable.
|
||||
4. A variety of forms are supported which are self-evident, but which allow users to have
|
||||
some flexibility in how they structure their YAML, JSON, or runtime collections.
|
||||
2. The Developer View of the ParsedOp API -- All op templates, regardless of the form they are
|
||||
provided in, are processed into a normalized internal data structure.
|
||||
1. The detailed documentation for the ParsedOp API is in javadoc.
|
||||
|
||||
The documentation in this directory serve as a testable specification for all the above. It
|
||||
shows specific examples of all the valid op template forms in both YAML and JSON, as well as how
|
||||
the data is normalized to feed developer's view of the ParsedOp API.
|
||||
|
||||
If you are a new user, it is recommended that you read the basic docs first before delving into
|
||||
these specification-level docs too much. The intro docs show normative and simple ways to
|
||||
specific workloads without worrying too much about all the possible forms.
|
||||
|
||||
## Templating Language
|
||||
|
||||
When users want to specify a set of operations to perform, they do so with the workload templating
|
||||
format, which includes document level details, block level details, and op level details.
|
||||
Specific reserved words like `block` or `ops` are used in tandem with nesting structure to
|
||||
define all valid workload constructions. Because of this, workload definitions are
|
||||
essentially data structures comprised of basic collection types and primitive values. Any on-disk
|
||||
format which can be loaded as such can be a valid source of workload definitions.
|
||||
|
||||
- [SpecTest Formatting](spectest_formatting.md) - A primer on the example formats used here
|
||||
- [Workload Structure](workload_structure.md) - Overall workload structure, keywords, nesting
|
||||
features
|
||||
- [Op Template Basics](op_template_basics.md) - Basic Details of op templating
|
||||
- [Op Template Variations](op_template_variations.md) - Additional op template variants
|
||||
and corner cases
|
||||
- [Template Variables](template_variables.md) - Textual macros and default values
|
||||
|
||||
## ParsedOp API
|
||||
|
||||
After a workload template is loaded into an activity, it is presented to the driver in an API which
|
||||
is suitable for building executable ops in the native driver.
|
||||
|
||||
- [ParsedOp API](parsed_op_api.md) - Defines the API which developers see after a workload is fully
|
||||
loaded.
|
||||
|
||||
## Related Reading
|
||||
|
||||
If you want to understand the rest of this document, it is crucial that you have a working knowledge
|
||||
of the standard YAML format and several examples from the current drivers. You can learn this from
|
||||
the main documentation which demonstrates step-by-step how to build a workload. Reading further in
|
||||
this document will be most useful for core NB developers, or advanced users who want to know all
|
||||
the possible ways of building workloads.
|
||||
|
||||
## Op Mapping Stages
|
||||
|
||||
The process of loading a workload definition occurs in several discrete steps during a NoSQLBench
|
||||
session:
|
||||
|
||||
1. The workload file is loaded.
|
||||
2. Template variables from the activity parameters are interposed into the raw contents of the
|
||||
file.
|
||||
3. The file is deserialized from its native form into a raw data structure.
|
||||
4. The raw data structure is transformed into a normalized data structure according to the Op
|
||||
Template normalization rules.
|
||||
5. Each op template is then denormalized as a self-contained data
|
||||
structure, containing all the provided bindings, params, and tags from the upper layers of the
|
||||
doc structure.
|
||||
6. The data is provided to the ParsedOp API for use by the developer.
|
||||
7. The DriverAdapter is loaded which understands the op fields provided in the op template.
|
||||
8. The DriverAdapter uses its documented rules to determine which types of native driver operations
|
||||
each op template is intended to represent. This is called **Op Mapping**.
|
||||
9. The DriverAdapter (via the selected Op Mapper) uses the identified types to create dispensers of
|
||||
native driver operations. This is called **Op Dispensing**.
|
||||
10. The op dispensers are arranged into an indexed bank of op sources according to the specified
|
||||
ratios and or sequencing strategy. From this point on, NoSQLBench has the ability to
|
||||
construct an operation for any given cycle at high speed.
|
||||
|
||||
These specifications are focused on steps 2-5. The DriverAdapter focuses on the developer's use of
|
||||
the ParsedOp API, and as such is documented in javadoc primarily. Some details on the ParsedOp
|
||||
API are shared here for basic awareness, but developers should look to the javadoc for the full
|
||||
story.
|
||||
|
||||
## Mapping vs Running
|
||||
|
||||
It should be noted that the Op Mapping stage, where user intentions are mapped from op templates to
|
||||
native operations is not something that needs to be done quickly. This occurs at
|
||||
_initialization_ time. Instead, it is more important to focus on user experience factors, such as
|
||||
flexibility, obviousness, robustness, correctness, and so on. Thus, priority of design factors in
|
||||
this part of NB is placed more on clear and purposeful abstractions and less on optimizing for
|
||||
speed. The clarity and detail which is conveyed by this layer to the driver developer will then
|
||||
enable them to focus on building fast and correct op dispensers. These dispensers are also
|
||||
constructed before the workload starts running, but are used at high speed while the workload
|
||||
is running.
|
@ -1,301 +0,0 @@
|
||||
# ParsedOp API
|
||||
|
||||
In the workload template examples, we show statements as being formed from a string value. This is a
|
||||
specific type of statement form, although it is possible to provide structured op templates as well.
|
||||
|
||||
**The ParsedOp API is responsible for converting all valid op template forms into a consistent and
|
||||
unambiguous model.** Thus, the rules for mapping the various forms to the command model must be
|
||||
precise. Those rules are the substance of this specification.
|
||||
|
||||
## Op Synthesis
|
||||
|
||||
The method of turning an op template, some data generation functions, and some seed values into an
|
||||
executable operation is called *Op Synthesis* in NoSQLBench. This is done in incremental stages:
|
||||
|
||||
1. During activity initialization, NoSQLBench parses the workload template and op templates
|
||||
contained within. Each active op template (after filtering) is converted to a parsed command.
|
||||
2. The NB driver uses the parsed command to guide the construction of an OpDispenser<T>. This is a
|
||||
dispenser of operations that can be executed by the driver's Action implementation.
|
||||
3. When it is time to create an actual operation to be executed, unique with its own procedurally
|
||||
generated payload and settings, the OpDispenser<T> is invoked as a LongFunction<T>. The input
|
||||
provided to this function is the cycle number of the operation. This is essentially a seed that
|
||||
determines the content of all the dynamic fields in the operation.
|
||||
|
||||
This process is non-trivial in that it is an incremental creational pattern, where the resultant
|
||||
object is contextual to some native API. The command API is intended to guide and enable op
|
||||
synthesis without tying developers' hands.
|
||||
|
||||
## Command Fields
|
||||
|
||||
A command structure is intended to provide all the fields needed to fully realize a native
|
||||
operation. Some of these fields will be constant, or *static* in the op template, expressed simply
|
||||
as strings, numbers, lists or maps. These are parsed from the op template as such and are cached in
|
||||
the command structure as statics.
|
||||
|
||||
Other fields are only prescribed as recipes. This comes in two parts: 1) The description for how to
|
||||
create the value from a binding function, and 2) the binding point within the op template. Suppose
|
||||
you have a string-based op template like this:
|
||||
|
||||
```yaml
|
||||
ops:
|
||||
- op1: select * from users where userid={userid}
|
||||
bindings:
|
||||
userid: ToString();
|
||||
```
|
||||
|
||||
In this case, there is only one op in the list of ops, having a name `op1` and a string form op
|
||||
template of `select * from users where userid={userid}`.
|
||||
|
||||
## Parsed Command Structure
|
||||
|
||||
Once an op template is parsed into a *parsed command*, it has the state shown in the data structure
|
||||
schematic below:
|
||||
|
||||
```json
|
||||
|
||||
{
|
||||
"name": "some-map-name",
|
||||
"statics": {
|
||||
"s1": "v1",
|
||||
"s2": {
|
||||
"f1": "valfoo"
|
||||
}
|
||||
},
|
||||
"dynamics": {
|
||||
"d1": "NumberNameToString()"
|
||||
},
|
||||
"captures": {
|
||||
"resultprop1": "asname1"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If either an **op** or **stmt** field is provided, then the same structure as above is used:
|
||||
|
||||
```json
|
||||
|
||||
{
|
||||
"name": "some-string-op",
|
||||
"statics": {
|
||||
},
|
||||
"dynamics": {
|
||||
"op": "select username from table where name userid={userid}"
|
||||
},
|
||||
"captures": {
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The parts of a parsed command structure are:
|
||||
|
||||
### command name
|
||||
|
||||
Each command knows its name, just like an op template does. This can be useful for diagnostics and
|
||||
metric naming.
|
||||
|
||||
### static fields
|
||||
|
||||
The field names which are statically assigned and their values of any type. Since these values are
|
||||
not generated per-op, they are kept separate as reference data. Knowing which fields are static and
|
||||
which are not makes it possible for developers to optimize op synthesis.
|
||||
|
||||
### dynamic fields
|
||||
|
||||
Named bindings points within the op template. These values will only be known for a given cycle.
|
||||
|
||||
### variable captures
|
||||
|
||||
Names of result values to save, and the variable names they are to be saved as. The names represent
|
||||
the name as it would be found in the native driver's API, such as the name `userid`
|
||||
in `select userid from ...`. In string form statements, users can specify that the userid should be
|
||||
saved as the thread-local variable named *userid* simply by tagging it
|
||||
like `select [userid] from ...`. They can also specify that this value should be captured under a
|
||||
different name with a variation like `select [userid as user_id] from ...`. This is the standard
|
||||
variable capture syntax for any string-based statement form.
|
||||
|
||||
# Resolved Command Structure
|
||||
|
||||
Once an op template has been parsed into a command structure, the runtime has everything it needs to
|
||||
know in order to realize a specific set of field values, *given a cycle number*. Within a cycle, the
|
||||
cycle number is effectively a seed value that drives the generation of all dynamic data for that
|
||||
cycle.
|
||||
|
||||
However, this seed value is only known by the runtime once it is time to execute a specific cycle.
|
||||
Thus, it is the developer's job to tell the NoSQLBench runtime how to map from the parsed structure
|
||||
to a native type of executable operation suitable for execution with that driver.
|
||||
|
||||
# Interpretation
|
||||
|
||||
A command structure does not necessarily describe a specific low-level operation to be performed by
|
||||
a native driver. It *should* do so, but it is up to the user to provide a valid op template
|
||||
according to the documented rules of op construction for that driver type. These rules should be
|
||||
clearly documented by the driver developer.
|
||||
|
||||
Once the command structure is provided, the driver takes over and maps the fields into an executable
|
||||
op -- *almost*. In fact, the driver developer defines the ways that a command structure can be
|
||||
turned into an executable operation. This is expressed as a *Function<CommandTemplate,T>* where T is
|
||||
the type used in the native driver's API.
|
||||
|
||||
How a developer maps a structure like the above to an operations is up to them. The general rule of
|
||||
thumb is to use the most obvious and familiar representation of an operation as it would appear to a
|
||||
user. If this is CQL or SQL, then recommend use that as the statement form. If it GraphQL, use that.
|
||||
In both of these cases, you have access to
|
||||
|
||||
## String Form
|
||||
|
||||
Basic operations are made from a statement in some type of query language:
|
||||
|
||||
```yaml
|
||||
ops:
|
||||
- stringform: select [userid] from db.users where user='{username}';
|
||||
bindings:
|
||||
username: NumberNameToString()
|
||||
```
|
||||
|
||||
## Structured Form
|
||||
|
||||
Some operations can't be easily represented by a single statement. Some operations are built from a
|
||||
set of fields which describe more about an operation than the basic statement form. These types of
|
||||
operations are expressed to NoSQLBench in map or *object* form, where the fields within the op can
|
||||
be specified independently.
|
||||
|
||||
```yaml
|
||||
ops:
|
||||
- structured1:
|
||||
stmt: select * from db.users where user='{username}}';
|
||||
prepared: true
|
||||
consistency_level: LOCAL_QUORUM
|
||||
bindings:
|
||||
username: NumberNameToString();
|
||||
- structured2:
|
||||
cmdtype: "put"
|
||||
namespace: users
|
||||
key: { userkey }
|
||||
body: "User42 was here"
|
||||
bindings:
|
||||
userkey: FixedValue(42)
|
||||
```
|
||||
|
||||
In the first case, the op named *structured1* is provided as a string value within a map structure.
|
||||
The *stmt* field is a reserved word (synonomous with op and operation). When you are reading an op
|
||||
from the command API, these will represented in exactly the same way as the stringform example
|
||||
above.
|
||||
|
||||
In the second case,
|
||||
|
||||
In the second, the op named *structured form* is provided as a map. Both of these examples would
|
||||
make sense to a user, as they are fairly self-explanatory.
|
||||
|
||||
Op templates may specify an op as either a string or a map. No other types are allowed. However,
|
||||
there are no restrictions placed on the elements below a map.
|
||||
|
||||
The driver developer should not have to parse all the possible structural forms that users can
|
||||
provide. There should be one way to access all of these in a consistent and unambiguous API.
|
||||
|
||||
## Command Structure
|
||||
|
||||
Here is an example data structure which illustrates all the possible elements of a parsed command:
|
||||
|
||||
```json
|
||||
|
||||
{
|
||||
"statics": {
|
||||
"prepared": "true",
|
||||
"consistency_level'"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Users provide a template form of an operation in each op template. This contains a sketch of what an
|
||||
operation might look like, and includes the following optional parts:
|
||||
|
||||
- properties of the operation, whether meta (like a statement) or payload content
|
||||
- the binding points where generated field values will be injected
|
||||
- The names of values to be extracted from the result of a successful operation.
|
||||
|
||||
## Statement Forms
|
||||
|
||||
Sometimes operations are derived from a query language, and are thus self-contained in a string
|
||||
form.
|
||||
|
||||
When mapping the template of an operation provided by users to an executable operation in some
|
||||
native driver with specific values, you have to know
|
||||
|
||||
* The s
|
||||
* The substance of the operation: The name and values of the fields that the user provides as part
|
||||
of the operation
|
||||
|
||||
Command templates are the third layer of workload templating. As described in other spec documents,
|
||||
the other layers are:
|
||||
|
||||
1. [Workload level templates](templated_workloads.md) - This specification covers the basics of a
|
||||
workload template, including the valid properties and structure.
|
||||
2. [Operation level templates](templated_operations.md) - This specification covers how operations
|
||||
can be specified, including semantics and structure.
|
||||
3. Command level templates, explained below. These are the detailed views of what goes into an op
|
||||
template, parsed and structured in a way that allows for efficient use at runtime.
|
||||
|
||||
Users do not create command templates directly. Instead, these are the *parsed* form of op templates
|
||||
as seen by the NB driver. The whole point of a command template is to provide crisp semantics and
|
||||
structure about what a user is asking a driver to do. Command Template
|
||||
|
||||
Command templates are essentially schematics for an operation. They are a structural interpretation
|
||||
of the content provided by users in op templates. Each op template provided can be converted into a
|
||||
command template. In short, the op template is the form that users tend to edit in yaml or provided
|
||||
as a data structure via scripting. **Command templates are the view of an op template as seen by an
|
||||
NB driver.**
|
||||
|
||||
```
|
||||
### Command Templates
|
||||
|
||||
Command templates are part of the workload API.
|
||||
|
||||
There exists a need to provide op templates to a myriad of runtime APIs,
|
||||
and thus it has to be flexible enough to serve them all.
|
||||
|
||||
1. In some cases, an operation is based on a query language where the
|
||||
query language itself encodes everything needed for specific operation.
|
||||
SQL queries are like this. This is a nice simplification, but it is not
|
||||
realistic for systems build on modern distributed principles.
|
||||
2. In most cases, you have both an operation and some qualifying rules
|
||||
about how the operation should be handled, such as consistency level.
|
||||
Thus, there is a need to provide parameters which can decorate
|
||||
operations.
|
||||
3. In some cases, you have a payload for your operation which is not based
|
||||
on a query language, but instead on an object with fields, or a verb
|
||||
which determines what other fields are needed. This structure is better
|
||||
described as a *command* than a *statement*.
|
||||
4. Finally, you must support separate both of the latter cases where the
|
||||
command or operations is defined in some pseudo-structured way, but it
|
||||
also has *separately* a set of qualifying parameters which are
|
||||
considered orthogonal, or at least separate from the meaning of the
|
||||
operation itself.
|
||||
|
||||
To address the full set of these mapping requirements, a type has been
|
||||
added to NB which provides a structured and pre-baked version of a
|
||||
resolvable command -- the CommandTemplate.
|
||||
|
||||
This type provides a view to the driver builder of all the fields
|
||||
specified by the user, whether as encoded as a string, such
|
||||
as `select row from ...`, or by a set of properties such
|
||||
as `{"verb":"get",
|
||||
"id":"2343"}`. It also exposes the parameters separately if provided.
|
||||
|
||||
### Static vs Dynamic command fields
|
||||
|
||||
Further, for each field in the command template, the driver implementor
|
||||
knows whether this was provided as a static value or one that can only be
|
||||
realized for a specific cycle (seed data). Thus, it is possible for
|
||||
advanced op mapping implementations to optimize the way that new
|
||||
operations are synthesized for efficiency.
|
||||
|
||||
For example, if you know that you have a command which has no dynamic
|
||||
fields in its command template, then it is possible to create a singleton
|
||||
op template which can simply be re-used. A fully dynamic command template,
|
||||
in contrast, may need to be realized dynamically for each cycle, given
|
||||
that you don't know the value of the fields in the command until you know
|
||||
the cycle value.
|
||||
|
||||
|
||||
|
||||
```
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,8 +16,8 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
@ -25,16 +25,16 @@ import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StatementsLoaderTest {
|
||||
public class OpsLoaderTest {
|
||||
|
||||
@Test
|
||||
public void testTemplateVarSubstitution() {
|
||||
StmtsDocList stmtsDocs = StatementsLoader.loadPath(null, "activities/template_vars", "src/test/resources");
|
||||
assertThat(stmtsDocs).isNotNull();
|
||||
List<StmtsDoc> docs = stmtsDocs.getStmtDocs();
|
||||
OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
|
||||
assertThat(opsDocs).isNotNull();
|
||||
List<OpsDoc> docs = opsDocs.getStmtDocs();
|
||||
assertThat(docs).hasSize(1);
|
||||
StmtsDoc stmtsBlocks = docs.get(0);
|
||||
Map<String, String> bindings = stmtsBlocks.getBindings();
|
||||
OpsDoc opsDoc = docs.get(0);
|
||||
Map<String, String> bindings = opsDoc.getBindings();
|
||||
assertThat(bindings).isEqualTo(Map.of(
|
||||
"b1a","Prefix(\"prefix\")",
|
||||
"b1b","Prefix(\"prefix\")",
|
||||
@ -46,7 +46,7 @@ public class StatementsLoaderTest {
|
||||
public void testInvalidYamlProperties() {
|
||||
Exception caught = null;
|
||||
try {
|
||||
StatementsLoader.loadPath(null, "activities/invalid_prop", "src/test/resources");
|
||||
OpsLoader.loadPath("activities/invalid_prop", Map.of(),"src/test/resources");
|
||||
} catch (Exception e) {
|
||||
caught = e;
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,11 +16,11 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.rawyaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsBlock;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsBlock;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -37,22 +37,22 @@ public class OpDefTest {
|
||||
@Test
|
||||
public void testLayering() {
|
||||
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
assertThat(all).isNotNull();
|
||||
assertThat(all.getStmtDocs()).hasSize(2);
|
||||
StmtsDoc doc1 = all.getStmtDocs().get(0);
|
||||
OpsDoc doc1 = all.getStmtDocs().get(0);
|
||||
assertThat(doc1.getName()).isEqualTo("doc1");
|
||||
assertThat(doc1.getBlocks()).hasSize(1);
|
||||
StmtsDoc doc2 = all.getStmtDocs().get(1);
|
||||
OpsDoc doc2 = all.getStmtDocs().get(1);
|
||||
assertThat(doc2.getBlocks()).hasSize(2);
|
||||
|
||||
StmtsBlock block1 = doc1.getBlocks().get(0);
|
||||
OpsBlock block1 = doc1.getBlocks().get(0);
|
||||
assertThat(block1.getBindings()).hasSize(2);
|
||||
assertThat(block1.getName()).isEqualTo("doc1--block0");
|
||||
assertThat(block1.getTags()).hasSize(1);
|
||||
|
||||
StmtsBlock block21 = doc2.getBlocks().get(0);
|
||||
StmtsBlock block22 = doc2.getBlocks().get(1);
|
||||
OpsBlock block21 = doc2.getBlocks().get(0);
|
||||
OpsBlock block22 = doc2.getBlocks().get(1);
|
||||
|
||||
assertThat(block21.getName()).isEqualTo("doc2--block1");
|
||||
assertThat(block21.getTags()).hasSize(3);
|
||||
@ -64,11 +64,11 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testStatementRendering() {
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
assertThat(all).isNotNull();
|
||||
assertThat(all.getStmtDocs()).hasSize(2);
|
||||
StmtsDoc doc1 = all.getStmtDocs().get(0);
|
||||
StmtsBlock block1 = doc1.getBlocks().get(0);
|
||||
OpsDoc doc1 = all.getStmtDocs().get(0);
|
||||
OpsBlock block1 = doc1.getBlocks().get(0);
|
||||
assertThat(block1.getName()).isEqualTo("doc1--block0");
|
||||
List<OpTemplate> ops = block1.getOps();
|
||||
assertThat(ops).hasSize(2);
|
||||
@ -79,10 +79,10 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testConsumableMapState() {
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
|
||||
List<StmtsDoc> docs = all.getStmtDocs();
|
||||
StmtsDoc block1 = docs.get(1);
|
||||
List<OpTemplate> stmts = block1.getStmts();
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc block1 = docs.get(1);
|
||||
List<OpTemplate> stmts = block1.getOpTemplates();
|
||||
OpTemplate stmt0 = stmts.get(0);
|
||||
OpTemplate stmt1 = stmts.get(1);
|
||||
assertThat(stmt0.getParams()).containsAllEntriesOf(Map.of("timeout", 23423, "foobar", "baz"));
|
||||
@ -94,12 +94,12 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testMapOfMaps() {
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
|
||||
List<StmtsDoc> docs = all.getStmtDocs();
|
||||
StmtsDoc doc0 = docs.get(0);
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc doc0 = docs.get(0);
|
||||
assertThat(doc0.getName()).isEqualTo("map-of-maps");
|
||||
assertThat(doc0.getBlocks()).hasSize(1);
|
||||
StmtsBlock block1 = doc0.getBlocks().get(0);
|
||||
OpsBlock block1 = doc0.getBlocks().get(0);
|
||||
assertThat(block1.getName()).isEqualTo("map-of-maps--block0");
|
||||
assertThat(block1.getOps()).hasSize(2);
|
||||
OpTemplate op0 = block1.getOps().get(0);
|
||||
@ -115,12 +115,12 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testBasicStringStmt() {
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
|
||||
List<StmtsDoc> docs = all.getStmtDocs();
|
||||
StmtsDoc doc1 = docs.get(1);
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc doc1 = docs.get(1);
|
||||
assertThat(doc1.getName()).isEqualTo("string-statement");
|
||||
assertThat(doc1.getBlocks()).hasSize(1);
|
||||
StmtsBlock block1 = doc1.getBlocks().get(0);
|
||||
OpsBlock block1 = doc1.getBlocks().get(0);
|
||||
assertThat(block1.getName()).isEqualTo("string-statement--block0");
|
||||
assertThat(block1.getOps()).hasSize(1);
|
||||
OpTemplate op0 = block1.getOps().get(0);
|
||||
@ -130,12 +130,12 @@ public class OpDefTest {
|
||||
|
||||
@Test
|
||||
public void testListOfNamedMap() {
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/statement_variants.yaml");
|
||||
List<StmtsDoc> docs = all.getStmtDocs();
|
||||
StmtsDoc doc2 = docs.get(2);
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/op_variants.yaml", Map.of());
|
||||
List<OpsDoc> docs = all.getStmtDocs();
|
||||
OpsDoc doc2 = docs.get(2);
|
||||
assertThat(doc2.getName()).isEqualTo("list-of-named-map");
|
||||
assertThat(doc2.getBlocks()).hasSize(1);
|
||||
StmtsBlock block1 = doc2.getBlocks().get(0);
|
||||
OpsBlock block1 = doc2.getBlocks().get(0);
|
||||
assertThat(block1.getOps()).hasSize(1);
|
||||
OpTemplate op0 = block1.getOps().get(0);
|
||||
System.out.println(op0.getParams());
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.rawyaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.*;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -24,26 +24,27 @@ import org.apache.logging.log4j.LogManager;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StmtEscapingTest {
|
||||
public class OpEscapingTest {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(StmtEscapingTest.class);
|
||||
private final static Logger logger = LogManager.getLogger(OpEscapingTest.class);
|
||||
private static List<OpTemplate> defs;
|
||||
|
||||
@BeforeAll
|
||||
public static void testLayering() {
|
||||
|
||||
StmtsDocList all = StatementsLoader.loadPath(logger, "testdocs/escaped_stmts.yaml");
|
||||
OpsDocList all = OpsLoader.loadPath("testdocs/escaped_ops.yaml", Map.of());
|
||||
assertThat(all).isNotNull();
|
||||
assertThat(all.getStmtDocs()).hasSize(1);
|
||||
StmtsDoc doc1 = all.getStmtDocs().get(0);
|
||||
OpsDoc doc1 = all.getStmtDocs().get(0);
|
||||
|
||||
// assertThat(doc1.getName()).isEqualTo("doc1");
|
||||
assertThat(doc1.getBlocks()).hasSize(1);
|
||||
|
||||
StmtsBlock block1 = doc1.getBlocks().get(0);
|
||||
OpsBlock block1 = doc1.getBlocks().get(0);
|
||||
assertThat(block1.getBindings()).hasSize(0);
|
||||
assertThat(block1.getTags()).hasSize(0);
|
||||
assertThat(block1.getOps()).hasSize(3);
|
@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.rawyaml;
|
||||
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class OpVariationTests {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(OpVariationTests.class);
|
||||
|
||||
@Test
|
||||
public void testListOpsOnly() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"ops:\n" +
|
||||
" - first op\n" +
|
||||
" - second op\n"
|
||||
);
|
||||
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(2);
|
||||
List<RawOpDef> ops = doc.getRawOpDefs();
|
||||
RawOpDef s0 = ops.get(0);
|
||||
assertThat(s0.getName()).isEqualTo("stmt1");
|
||||
assertThat(s0.getStmt()).isEqualTo("first op");
|
||||
RawOpDef s1 = ops.get(1);
|
||||
assertThat(s1.getName()).isEqualTo("stmt2");
|
||||
assertThat(s1.getStmt()).isEqualTo("second op");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleEntryMapStmtsOnly() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"ops:\n" +
|
||||
" - s1: op one\n" +
|
||||
" - s2: op two\n"
|
||||
);
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(2);
|
||||
List<RawOpDef> ops = doc.getRawOpDefs();
|
||||
assertThat(ops.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(ops.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(ops.get(0).getStmt()).isEqualTo("op one");
|
||||
assertThat(ops.get(1)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(ops.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(ops.get(1).getStmt()).isEqualTo("op two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapStmtsOnly() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"ops:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: op one\n" +
|
||||
" - name: s2\n" +
|
||||
" stmt: op two\n"
|
||||
);
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(2);
|
||||
List<RawOpDef> stmts = doc.getRawOpDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("op one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("op two");
|
||||
}
|
||||
|
||||
/**
|
||||
* This test uses the compatible names for the sake of demonstration. Users should know that they can use op and
|
||||
* statement terms interchangeably in op templates.
|
||||
*/
|
||||
@Test
|
||||
public void testMixedForms() {
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList docs = ysl.loadString(
|
||||
"statement:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: statement one\n" +
|
||||
" - statement two\n" +
|
||||
" - s3: statement three\n" +
|
||||
" - ST4: statement four\n" +
|
||||
" type: organic\n"
|
||||
);
|
||||
assertThat(docs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc doc = docs.getOpsDocs().get(0);
|
||||
assertThat(doc.getRawOpDefs()).hasSize(4);
|
||||
List<RawOpDef> stmts = doc.getRawOpDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("stmt2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
|
||||
assertThat(stmts.get(2)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(2).getName()).isEqualTo("s3");
|
||||
assertThat(stmts.get(2).getStmt()).isEqualTo("statement three");
|
||||
assertThat(stmts.get(3)).isOfAnyClassIn(RawOpDef.class);
|
||||
assertThat(stmts.get(3).getName()).isEqualTo("ST4");
|
||||
assertThat(stmts.get(3).getStmt()).isEqualTo("statement four");
|
||||
assertThat(stmts.get(3).getParams().get("type")).isEqualTo("organic");
|
||||
|
||||
}
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -25,16 +25,16 @@ import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class RawYamlStatementLoaderTest {
|
||||
private final static Logger logger = LogManager.getLogger(RawYamlStatementLoaderTest.class);
|
||||
public class RawYamlTemplateLoaderTest {
|
||||
private final static Logger logger = LogManager.getLogger(RawYamlTemplateLoaderTest.class);
|
||||
|
||||
@Test
|
||||
public void testLoadPropertiesBlock() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList rawBlockDocs = ysl.loadPath(logger, "testdocs/rawblock.yaml");
|
||||
assertThat(rawBlockDocs.getStmtsDocs()).hasSize(1);
|
||||
RawStmtsDoc rawBlockDoc = rawBlockDocs.getStmtsDocs().get(0);
|
||||
assertThat(rawBlockDoc.getRawStmtDefs()).hasSize(1);
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList rawBlockDocs = ysl.loadPath("testdocs/rawblock.yaml");
|
||||
assertThat(rawBlockDocs.getOpsDocs()).hasSize(1);
|
||||
RawOpsDoc rawBlockDoc = rawBlockDocs.getOpsDocs().get(0);
|
||||
assertThat(rawBlockDoc.getRawOpDefs()).hasSize(1);
|
||||
assertThat(rawBlockDoc.getBindings()).hasSize(1);
|
||||
assertThat(rawBlockDoc.getName()).isEqualTo("name");
|
||||
assertThat(rawBlockDoc.getTags()).hasSize(1);
|
||||
@ -43,30 +43,30 @@ public class RawYamlStatementLoaderTest {
|
||||
|
||||
@Test
|
||||
public void testLoadFullFormat() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
|
||||
List<RawStmtsDoc> rawStmtsDocs = erthing.getStmtsDocs();
|
||||
assertThat(rawStmtsDocs).hasSize(2);
|
||||
RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0);
|
||||
List<RawStmtsBlock> blocks = rawStmtsDoc.getBlocks();
|
||||
assertThat(rawStmtsDoc.getName()).isEqualTo("doc1");
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
|
||||
List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
|
||||
assertThat(rawOpsDocs).hasSize(2);
|
||||
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
|
||||
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
|
||||
assertThat(rawOpsDoc.getName()).isEqualTo("doc1");
|
||||
assertThat(blocks).hasSize(1);
|
||||
RawStmtsBlock rawStmtsBlock = blocks.get(0);
|
||||
assertThat(rawStmtsBlock.getName()).isEqualTo("block0");
|
||||
RawOpsBlock rawOpsBlock = blocks.get(0);
|
||||
assertThat(rawOpsBlock.getName()).isEqualTo("block0");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadScenarios() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
|
||||
List<RawStmtsDoc> rawStmtsDocs = erthing.getStmtsDocs();
|
||||
assertThat(rawStmtsDocs).hasSize(2);
|
||||
RawStmtsDoc rawStmtsDoc = rawStmtsDocs.get(0);
|
||||
List<RawStmtsBlock> blocks = rawStmtsDoc.getBlocks();
|
||||
assertThat(rawStmtsDoc.getDesc()).isEqualTo(
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/docs_blocks_ops.yaml");
|
||||
List<RawOpsDoc> rawOpsDocs = erthing.getOpsDocs();
|
||||
assertThat(rawOpsDocs).hasSize(2);
|
||||
RawOpsDoc rawOpsDoc = rawOpsDocs.get(0);
|
||||
List<RawOpsBlock> blocks = rawOpsDoc.getBlocks();
|
||||
assertThat(rawOpsDoc.getDesc()).isEqualTo(
|
||||
"a quintessential description - this is superseded by dedicated specification tests and will be removed");
|
||||
|
||||
RawScenarios rawScenarios = rawStmtsDoc.getRawScenarios();
|
||||
RawScenarios rawScenarios = rawOpsDoc.getRawScenarios();
|
||||
assertThat(rawScenarios.getScenarioNames()).containsExactly("default", "schema-only");
|
||||
Map<String, String> defaultScenario = rawScenarios.getNamedScenario("default");
|
||||
assertThat(defaultScenario.keySet())
|
||||
@ -79,16 +79,16 @@ public class RawYamlStatementLoaderTest {
|
||||
assertThat(schemaOnlyScenario.values())
|
||||
.containsExactly("run driver=blah tags=phase:schema");
|
||||
|
||||
assertThat(rawStmtsDoc.getName()).isEqualTo("doc1");
|
||||
assertThat(rawOpsDoc.getName()).isEqualTo("doc1");
|
||||
assertThat(blocks).hasSize(1);
|
||||
RawStmtsBlock rawStmtsBlock = blocks.get(0);
|
||||
assertThat(rawStmtsBlock.getName()).isEqualTo("block0");
|
||||
RawOpsBlock rawOpsBlock = blocks.get(0);
|
||||
assertThat(rawOpsBlock.getName()).isEqualTo("block0");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testErrorMsg() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList erthing = ysl.loadPath(logger, "testdocs/badyamlfile.yaml");
|
||||
RawOpsLoader ysl = new RawOpsLoader();
|
||||
RawOpsDocList erthing = ysl.loadPath("testdocs/badyamlfile.yaml");
|
||||
}
|
||||
|
||||
}
|
@ -1,125 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.rawyaml;
|
||||
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StmtVariationTests {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(StmtVariationTests.class);
|
||||
|
||||
@Test
|
||||
public void testListStmtsOnly() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList docs = ysl.loadString(logger,
|
||||
"statements:\n" +
|
||||
" - first statement\n" +
|
||||
" - second statement\n"
|
||||
);
|
||||
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(2);
|
||||
List<RawStmtDef> stmts = doc.getRawStmtDefs();
|
||||
RawStmtDef s0 = stmts.get(0);
|
||||
assertThat(s0.getName()).isEqualTo("stmt1");
|
||||
assertThat(s0.getStmt()).isEqualTo("first statement");
|
||||
RawStmtDef s1 = stmts.get(1);
|
||||
assertThat(s1.getName()).isEqualTo("stmt2");
|
||||
assertThat(s1.getStmt()).isEqualTo("second statement");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSingleEntryMapStmtsOnly() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList docs = ysl.loadString(logger,
|
||||
"statements:\n" +
|
||||
" - s1: statement one\n" +
|
||||
" - s2: statement two\n"
|
||||
);
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(2);
|
||||
List<RawStmtDef> stmts = doc.getRawStmtDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMapStmtsOnly() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList docs = ysl.loadString(logger,
|
||||
"statements:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: statement one\n" +
|
||||
" - name: s2\n" +
|
||||
" stmt: statement two\n"
|
||||
);
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(2);
|
||||
List<RawStmtDef> stmts = doc.getRawStmtDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("s2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMixedForms() {
|
||||
RawStmtsLoader ysl = new RawStmtsLoader();
|
||||
RawStmtsDocList docs = ysl.loadString(logger,
|
||||
"statement:\n" +
|
||||
" - name: s1\n" +
|
||||
" stmt: statement one\n" +
|
||||
" - statement two\n" +
|
||||
" - s3: statement three\n" +
|
||||
" - ST4: statement four\n" +
|
||||
" type: organic\n"
|
||||
);
|
||||
assertThat(docs.getStmtsDocs()).hasSize(1);
|
||||
RawStmtsDoc doc = docs.getStmtsDocs().get(0);
|
||||
assertThat(doc.getRawStmtDefs()).hasSize(4);
|
||||
List<RawStmtDef> stmts = doc.getRawStmtDefs();
|
||||
assertThat(stmts.get(0)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(0).getName()).isEqualTo("s1");
|
||||
assertThat(stmts.get(0).getStmt()).isEqualTo("statement one");
|
||||
assertThat(stmts.get(1)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(1).getName()).isEqualTo("stmt2");
|
||||
assertThat(stmts.get(1).getStmt()).isEqualTo("statement two");
|
||||
assertThat(stmts.get(2)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(2).getName()).isEqualTo("s3");
|
||||
assertThat(stmts.get(2).getStmt()).isEqualTo("statement three");
|
||||
assertThat(stmts.get(3)).isOfAnyClassIn(RawStmtDef.class);
|
||||
assertThat(stmts.get(3).getName()).isEqualTo("ST4");
|
||||
assertThat(stmts.get(3).getStmt()).isEqualTo("statement four");
|
||||
assertThat(stmts.get(3).getParams().get("type")).isEqualTo("organic");
|
||||
|
||||
}
|
||||
}
|
@ -18,10 +18,11 @@ package io.nosqlbench.engine.api.activityconfig.specifications;
|
||||
|
||||
import com.google.gson.*;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.nb.spectest.api.STAssemblyValidator;
|
||||
import io.nosqlbench.nb.spectest.core.STNodeAssembly;
|
||||
import io.nosqlbench.nb.spectest.loaders.STDefaultLoader;
|
||||
@ -107,8 +108,8 @@ public class YamlSpecValidator implements STAssemblyValidator {
|
||||
}.getType();
|
||||
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
|
||||
|
||||
StmtsDocList stmtsDocs = StatementsLoader.loadString(yaml, Map.of());
|
||||
List<OpTemplate> stmts = stmtsDocs.getStmts();
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
|
||||
List<OpTemplate> stmts = stmtsDocs.getOps();
|
||||
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());
|
||||
|
||||
try {
|
||||
@ -142,8 +143,6 @@ public class YamlSpecValidator implements STAssemblyValidator {
|
||||
private void validateYamlWithJson(String desc, String yaml, String json, STNodeReference testset, boolean debug) {
|
||||
System.out.format("%-40s", "- checking yaml->json");
|
||||
|
||||
// StmtsDocList stmts = StatementsLoader.loadString(yaml);
|
||||
|
||||
try {
|
||||
List<Map<String, Object>> docmaps = new RawYamlLoader().loadString(logger, yaml);
|
||||
JsonElement elem = null;
|
||||
@ -174,10 +173,9 @@ public class YamlSpecValidator implements STAssemblyValidator {
|
||||
throw new RuntimeException("unknown type in comparator: " + json);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("error while verifying model:\n");
|
||||
sb.append(" path: ").append(testset.getPath().toString()).append("\n");
|
||||
sb.append(" line: ").append(testset.getLineNumber()).append("\n");
|
||||
String sb = "error while verifying model:\n" +
|
||||
" path: " + testset.getPath().toString() + "\n" +
|
||||
" line: " + testset.getLineNumber() + "\n";
|
||||
|
||||
logger.error(sb + ": " + e.getMessage(), e);
|
||||
throw new RuntimeException(e);
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -26,21 +26,21 @@ import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StmtDetailOverrideTest {
|
||||
private static final Logger logger = LogManager.getLogger(StmtDetailOverrideTest.class);
|
||||
public class OpDetailOverrideTest {
|
||||
private static final Logger logger = LogManager.getLogger(OpDetailOverrideTest.class);
|
||||
|
||||
@Test
|
||||
public void testStmtOverrides() {
|
||||
|
||||
StmtsDocList doclist = StatementsLoader.loadPath(logger, "testdocs/stmt_details.yaml");
|
||||
OpsDocList doclist = OpsLoader.loadPath("testdocs/op_details.yaml", Map.of());
|
||||
|
||||
assertThat(doclist).isNotNull();
|
||||
|
||||
assertThat(doclist.getStmtDocs()).hasSize(1);
|
||||
StmtsDoc doc1 = doclist.getStmtDocs().get(0);
|
||||
OpsDoc doc1 = doclist.getStmtDocs().get(0);
|
||||
|
||||
assertThat(doc1.getBlocks()).hasSize(2);
|
||||
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
|
||||
OpsBlock doc1block0 = doc1.getBlocks().get(0);
|
||||
assertThat(doc1block0.getOps().size()).isEqualTo(1);
|
||||
OpTemplate s = doc1block0.getOps().get(0);
|
||||
assertThat(s.getName()).isEqualTo("block0--stmt1");
|
||||
@ -49,25 +49,25 @@ public class StmtDetailOverrideTest {
|
||||
assertThat(s.getParams()).hasSize(1);
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","block0","global_tag1","tag value","name","block0--stmt1"));
|
||||
|
||||
StmtsBlock doc1block1 = doc1.getBlocks().get(1);
|
||||
List<OpTemplate> stmts = doc1block1.getOps();
|
||||
assertThat(stmts).hasSize(4);
|
||||
OpsBlock doc1block1 = doc1.getBlocks().get(1);
|
||||
List<OpTemplate> ops = doc1block1.getOps();
|
||||
assertThat(ops).hasSize(4);
|
||||
|
||||
s = stmts.get(0);
|
||||
s = ops.get(0);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--stmt1");
|
||||
assertThat(s.getStmt()).contains("astatement1");
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--stmt1"));
|
||||
assertThat(s.getBindings()).hasSize(1);
|
||||
assertThat(s.getParams()).hasSize(1);
|
||||
|
||||
s = stmts.get(1);
|
||||
s = ops.get(1);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--s2name");
|
||||
assertThat(s.getStmt()).contains("s2statement data");
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s2name"));
|
||||
assertThat(s.getBindings()).hasSize(1);
|
||||
assertThat(s.getParams()).hasSize(1);
|
||||
|
||||
s = stmts.get(2);
|
||||
s = ops.get(2);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--s3");
|
||||
assertThat(s.getStmt()).contains("statement three");
|
||||
assertThat(s.getTags()).containsEntry("tname1", "tval1");
|
||||
@ -75,7 +75,7 @@ public class StmtDetailOverrideTest {
|
||||
assertThat(s.getBindings()).hasSize(3);
|
||||
assertThat(s.getParams()).hasSize(2);
|
||||
|
||||
s = stmts.get(3);
|
||||
s = ops.get(3);
|
||||
assertThat(s.getName()).isEqualTo("testblock1--s4");
|
||||
assertThat(s.getStmt()).contains("statement 4");
|
||||
assertThat(s.getTags()).isEqualTo(Map.of("block","testblock1","global_tag1","tag value","name","testblock1--s4"));
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import org.assertj.core.data.MapEntry;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -29,10 +29,10 @@ import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class StmtsDocListTest {
|
||||
public class OpsDocListTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(StmtsDocListTest.class);
|
||||
private static StmtsDocList doclist;
|
||||
private static final Logger logger = LogManager.getLogger(OpsDocListTest.class);
|
||||
private static OpsDocList doclist;
|
||||
|
||||
private final LinkedHashMap<String, String> doc0bindings = new LinkedHashMap<String, String>() {{
|
||||
put("b2", "b2d");
|
||||
@ -50,17 +50,17 @@ public class StmtsDocListTest {
|
||||
|
||||
@BeforeAll
|
||||
public static void testLoadYaml() {
|
||||
doclist = StatementsLoader.loadPath(logger, "testdocs/docs_blocks_stmts.yaml");
|
||||
doclist = OpsLoader.loadPath("testdocs/docs_blocks_ops.yaml", Map.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlocksInheritDocData() {
|
||||
assertThat(doclist).isNotNull();
|
||||
assertThat(doclist.getStmtDocs()).hasSize(2);
|
||||
StmtsDoc doc1 = doclist.getStmtDocs().get(0);
|
||||
OpsDoc doc1 = doclist.getStmtDocs().get(0);
|
||||
|
||||
assertThat(doc1.getBlocks()).hasSize(1);
|
||||
StmtsBlock doc1block0 = doc1.getBlocks().get(0);
|
||||
OpsBlock doc1block0 = doc1.getBlocks().get(0);
|
||||
|
||||
assertThat(doc1.getTags()).isEqualTo(doc0tags);
|
||||
assertThat(doc1.getBindings()).isEqualTo(doc0bindings);
|
||||
@ -75,29 +75,29 @@ public class StmtsDocListTest {
|
||||
|
||||
@Test
|
||||
public void testStmtInheritsBlockData() {
|
||||
StmtsDoc doc0 = doclist.getStmtDocs().get(0);
|
||||
List<OpTemplate> stmts1 = doc0.getBlocks().get(0).getOps();
|
||||
assertThat(stmts1).hasSize(2);
|
||||
OpsDoc doc0 = doclist.getStmtDocs().get(0);
|
||||
List<OpTemplate> ops1 = doc0.getBlocks().get(0).getOps();
|
||||
assertThat(ops1).hasSize(2);
|
||||
|
||||
StmtsBlock block0 = doc0.getBlocks().get(0);
|
||||
OpsBlock block0 = doc0.getBlocks().get(0);
|
||||
assertThat(block0.getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(block0.getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(block0.getTags()).containsExactly(MapEntry.entry("atagname","atagvalue"));
|
||||
|
||||
assertThat(stmts1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(stmts1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(stmts1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
|
||||
assertThat(ops1.get(0).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(ops1.get(0).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(ops1.get(0).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt1","block","doc1--block0"));
|
||||
|
||||
assertThat(stmts1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(stmts1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(stmts1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
|
||||
assertThat(ops1.get(1).getBindings()).containsExactly(MapEntry.entry("b2","b2d"),MapEntry.entry("b1","b1d"));
|
||||
assertThat(ops1.get(1).getParams()).containsExactly(MapEntry.entry("param1","value1"));
|
||||
assertThat(ops1.get(1).getTags()).isEqualTo(Map.of("atagname","atagvalue","name","doc1--block0--stmt2","block","doc1--block0"));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlockLayersDocData() {
|
||||
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
|
||||
StmtsBlock block0 = doc1.getBlocks().get(0);
|
||||
OpsDoc doc1 = doclist.getStmtDocs().get(1);
|
||||
OpsBlock block0 = doc1.getBlocks().get(0);
|
||||
|
||||
Map<String, String> doc1block0tags = block0.getTags();
|
||||
Map<String, String> doc1block0params = block0.getParamsAsText();
|
||||
@ -119,16 +119,16 @@ public class StmtsDocListTest {
|
||||
|
||||
@Test
|
||||
public void testStmtsGetter() {
|
||||
StmtsDoc doc1 = doclist.getStmtDocs().get(1);
|
||||
List<OpTemplate> stmts = doc1.getStmts();
|
||||
OpsDoc doc1 = doclist.getStmtDocs().get(1);
|
||||
List<OpTemplate> stmts = doc1.getOpTemplates();
|
||||
assertThat(stmts).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFilteredStmts() {
|
||||
List<OpTemplate> stmts = doclist.getStmts("");
|
||||
List<OpTemplate> stmts = doclist.getOps("");
|
||||
assertThat(stmts).hasSize(6);
|
||||
stmts = doclist.getStmts("root1:value23");
|
||||
stmts = doclist.getOps("root1:value23");
|
||||
assertThat(stmts).hasSize(2);
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,33 +16,35 @@
|
||||
|
||||
package io.nosqlbench.engine.api.activityconfig.yaml;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class ParsedStmtOpTest {
|
||||
private static final Logger logger = LogManager.getLogger(ParsedStmtOpTest.class);
|
||||
private static StmtsDocList doclist;
|
||||
public class ParsedOpTemplateTest {
|
||||
private static final Logger logger = LogManager.getLogger(ParsedOpTemplateTest.class);
|
||||
private static OpsDocList doclist;
|
||||
|
||||
@BeforeAll
|
||||
public static void testLoadYaml() {
|
||||
doclist = StatementsLoader.loadPath(logger, "testdocs/bindings.yaml");
|
||||
doclist = OpsLoader.loadPath("testdocs/bindings.yaml", Map.of());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasicParser() {
|
||||
StmtsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
|
||||
OpsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
|
||||
OpTemplate stmtDef0 = block0.getOps().get(0);
|
||||
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
|
||||
assertThat(parsed0.getMissing()).containsExactly("delta");
|
||||
assertThat(parsed0.hasError()).isTrue();
|
||||
|
||||
StmtsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
|
||||
OpsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
|
||||
OpTemplate stmtDef1 = block1.getOps().get(0);
|
||||
ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
|
||||
assertThat(parsed1.getMissing()).containsExactly();
|
||||
@ -51,7 +53,7 @@ public class ParsedStmtOpTest {
|
||||
|
||||
@Test
|
||||
public void testMultipleBindingUsage() {
|
||||
StmtsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
|
||||
OpsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
|
||||
|
||||
OpTemplate stmtDef0 = block2.getOps().get(0);
|
||||
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,10 +16,11 @@
|
||||
|
||||
package io.nosqlbench.engine.api.templating;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpData;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.api.config.standard.ConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.api.config.standard.Param;
|
||||
@ -68,9 +69,9 @@ public class ParsedOpTest {
|
||||
params:
|
||||
ps1: "param-one"
|
||||
""";
|
||||
StmtsDocList stmtsDocs = StatementsLoader.loadString(opt, cfg.getMap());
|
||||
assertThat(stmtsDocs.getStmts().size()).isEqualTo(1);
|
||||
OpTemplate opTemplate = stmtsDocs.getStmts().get(0);
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString(opt, OpTemplateFormat.yaml, cfg.getMap(), null);
|
||||
assertThat(stmtsDocs.getOps().size()).isEqualTo(1);
|
||||
OpTemplate opTemplate = stmtsDocs.getOps().get(0);
|
||||
ParsedOp parsedOp = new ParsedOp(opTemplate, cfg);
|
||||
|
||||
assertThat(parsedOp.getAsFunctionOr("d1","invalid").apply(1L)).isEqualTo("one");
|
||||
|
@ -94,7 +94,7 @@
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.media</groupId>
|
||||
<artifactId>jersey-media-json-jackson</artifactId>
|
||||
<version>3.1.0</version>
|
||||
<version>3.1.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@ -133,51 +133,4 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
@ -47,51 +47,4 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
@ -41,54 +41,4 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -42,52 +42,5 @@
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -65,54 +65,6 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>confluent</id>
|
||||
|
@ -90,52 +90,5 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -136,51 +136,6 @@
|
||||
<filtering>false</filtering> <!-- exclusion from defaults -->
|
||||
</testResource>
|
||||
</testResources>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
|
@ -37,10 +37,10 @@ import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
|
||||
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiter;
|
||||
import io.nosqlbench.engine.api.activityapi.ratelimits.RateLimiters;
|
||||
import io.nosqlbench.engine.api.activityapi.ratelimits.RateSpec;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.motor.RunStateTally;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DryRunOpDispenserWrapper;
|
||||
@ -109,7 +109,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initActivity() {
|
||||
public synchronized void initActivity() {
|
||||
initOrUpdateRateLimiters(this.activityDef);
|
||||
}
|
||||
|
||||
@ -353,7 +353,7 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
*
|
||||
* @param seq - The {@link OpSequence} to derive the defaults from
|
||||
*/
|
||||
public void setDefaultsFromOpSequence(OpSequence<?> seq) {
|
||||
public synchronized void setDefaultsFromOpSequence(OpSequence<?> seq) {
|
||||
Optional<String> strideOpt = getParams().getOptionalString("stride");
|
||||
if (strideOpt.isEmpty()) {
|
||||
String stride = String.valueOf(seq.getSequence().length);
|
||||
@ -549,21 +549,21 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
|
||||
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
|
||||
|
||||
StmtsDocList stmtsDocList = loadStmtsDocList();
|
||||
OpsDocList opsDocList = loadStmtsDocList();
|
||||
|
||||
List<OpTemplate> unfilteredOps = stmtsDocList.getStmts();
|
||||
List<OpTemplate> filteredOps = stmtsDocList.getStmts(tagfilter);
|
||||
List<OpTemplate> unfilteredOps = opsDocList.getOps();
|
||||
List<OpTemplate> filteredOps = opsDocList.getOps(tagfilter);
|
||||
|
||||
if (filteredOps.size() == 0) {
|
||||
if (unfilteredOps.size() > 0) { // There were no ops, and it was because they were all filtered out
|
||||
throw new BasicError("There were no active statements with tag filter '"
|
||||
throw new BasicError("There were no active op templates with tag filter '"
|
||||
+ tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
|
||||
} else {
|
||||
// There were no ops, and it *wasn't* because they were all filtered out.
|
||||
|
||||
// In this case, let's try to synthesize the ops as long as at least a default driver was provided
|
||||
if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
|
||||
filteredOps = sotp.getSyntheticOpTemplates(stmtsDocList, getActivityDef().getParams());
|
||||
filteredOps = sotp.getSyntheticOpTemplates(opsDocList, getActivityDef().getParams());
|
||||
Objects.requireNonNull(filteredOps);
|
||||
if (filteredOps.size() == 0) {
|
||||
throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + "'" +
|
||||
@ -573,12 +573,12 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
throw new BasicError("""
|
||||
No op templates were provided. You must provide one of these activity parameters:
|
||||
1) workload=some.yaml
|
||||
2) op='inline template
|
||||
2) op='inline template'
|
||||
3) driver=stdout (or any other drive that can synthesize ops)""");
|
||||
}
|
||||
}
|
||||
if (filteredOps.size() == 0) {
|
||||
throw new BasicError("There were no active statements with tag filter '" + tagfilter + "'");
|
||||
throw new BasicError("There were no active op templates with tag filter '" + tagfilter + "'");
|
||||
}
|
||||
}
|
||||
|
||||
@ -613,8 +613,8 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
* taken as the only provided statement.</LI>
|
||||
* <LI>If a 'yaml, or 'workload' parameter is provided, then the statements in that file
|
||||
* are taken with their ratios </LI>
|
||||
* <LI>Any provided tags filter is used to select only the statements which have matching
|
||||
* tags. If no tags are provided, then all the found statements are included.</LI>
|
||||
* <LI>Any provided tags filter is used to select only the op templates which have matching
|
||||
* tags. If no tags are provided, then all the found op templates are included.</LI>
|
||||
* <LI>The ratios and the 'seq' parameter are used to build a sequence of the ready operations,
|
||||
* where the sequence length is the sum of the ratios.</LI>
|
||||
* </OL>
|
||||
@ -661,20 +661,20 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs
|
||||
return planner.resolve();
|
||||
}
|
||||
|
||||
protected StmtsDocList loadStmtsDocList() {
|
||||
protected OpsDocList loadStmtsDocList() {
|
||||
|
||||
try {
|
||||
Optional<String> stmt = activityDef.getParams().getOptionalString("op", "stmt", "statement");
|
||||
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
|
||||
if (stmt.isPresent()) {
|
||||
workloadSource = "commandline:" + stmt.get();
|
||||
return StatementsLoader.loadStmt(logger, stmt.get(), activityDef.getParams());
|
||||
return OpsLoader.loadString(stmt.get(), OpTemplateFormat.inline, activityDef.getParams(), null);
|
||||
} else if (op_yaml_loc.isPresent()) {
|
||||
workloadSource = "yaml:" + op_yaml_loc.get();
|
||||
return StatementsLoader.loadPath(logger, op_yaml_loc.get(), activityDef.getParams(), "activities");
|
||||
return OpsLoader.loadPath(op_yaml_loc.get(), activityDef.getParams(), "activities");
|
||||
}
|
||||
|
||||
return StmtsDocList.none();
|
||||
return OpsDocList.none();
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -33,6 +33,7 @@ import io.nosqlbench.engine.api.activityimpl.MotorState;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static io.nosqlbench.engine.api.activityapi.core.RunState.*;
|
||||
@ -460,18 +461,17 @@ public class CoreMotor<D> implements ActivityDefObserver, Motor<D>, Stoppable {
|
||||
|
||||
@Override
|
||||
public synchronized void requestStop() {
|
||||
if (motorState.get() == Running) {
|
||||
RunState currentState = motorState.get();
|
||||
if (Objects.requireNonNull(currentState) == Running) {
|
||||
if (input instanceof Stoppable) {
|
||||
((Stoppable) input).requestStop();
|
||||
}
|
||||
if (action instanceof Stoppable) {
|
||||
((Stoppable) action).requestStop();
|
||||
}
|
||||
motorState.enterState(RunState.Stopping);
|
||||
motorState.enterState(Stopping);
|
||||
} else {
|
||||
if (motorState.get() != Stopped && motorState.get() != Stopping) {
|
||||
logger.warn(()->"attempted to stop motor " + this.getSlotId() + ": from non Running state:" + motorState.get());
|
||||
}
|
||||
logger.warn(() -> "attempted to stop motor " + this.getSlotId() + ": from non Running state:" + currentState);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -20,9 +20,9 @@ import io.nosqlbench.api.config.standard.*;
|
||||
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.api.errors.OpConfigError;
|
||||
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
|
||||
@ -54,11 +54,12 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
|
||||
|
||||
public StandardActivity(ActivityDef activityDef) {
|
||||
super(activityDef);
|
||||
OpsDocList workload;
|
||||
|
||||
Optional<String> yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
|
||||
if (yaml_loc.isPresent()) {
|
||||
Map<String, Object> disposable = new LinkedHashMap<>(activityDef.getParams());
|
||||
StmtsDocList workload = StatementsLoader.loadPath(logger, yaml_loc.get(), disposable, "activities");
|
||||
workload = OpsLoader.loadPath(yaml_loc.get(), disposable, "activities");
|
||||
yamlmodel = workload.getConfigModel();
|
||||
} else {
|
||||
yamlmodel = ConfigModel.of(StandardActivity.class).asReadOnly();
|
||||
@ -163,11 +164,11 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<OpTemplate> getSyntheticOpTemplates(StmtsDocList stmtsDocList, Map<String, Object> cfg) {
|
||||
public List<OpTemplate> getSyntheticOpTemplates(OpsDocList opsDocList, Map<String, Object> cfg) {
|
||||
List<OpTemplate> opTemplates = new ArrayList<>();
|
||||
for (DriverAdapter adapter : adapters.values()) {
|
||||
if (adapter instanceof SyntheticOpTemplateProvider sotp) {
|
||||
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(stmtsDocList, cfg);
|
||||
List<OpTemplate> newTemplates = sotp.getSyntheticOpTemplates(opsDocList, cfg);
|
||||
opTemplates.addAll(newTemplates);
|
||||
}
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -18,8 +18,8 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
|
||||
|
||||
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
|
||||
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
|
||||
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
|
||||
import io.nosqlbench.api.config.standard.NBConfigModel;
|
||||
@ -69,7 +69,7 @@ public class StandardActivityType<A extends StandardActivity<?,?>> extends Simpl
|
||||
Optional<String> op_yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload");
|
||||
if (op_yaml_loc.isPresent()) {
|
||||
Map<String,Object> disposable = new LinkedHashMap<>(activityDef.getParams());
|
||||
StmtsDocList workload = StatementsLoader.loadPath(logger, op_yaml_loc.get(), disposable, "activities");
|
||||
OpsDocList workload = OpsLoader.loadPath(op_yaml_loc.get(), disposable, "activities");
|
||||
cfgModel=cfgModel.add(workload.getConfigModel());
|
||||
}
|
||||
NBConfiguration cfg = cfgModel.apply(activityDef.getParams());
|
||||
|
@ -16,10 +16,10 @@
|
||||
|
||||
package io.nosqlbench.engine.api.scenarios;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.templating.StrInterpolator;
|
||||
import io.nosqlbench.api.content.Content;
|
||||
import io.nosqlbench.api.content.NBIO;
|
||||
@ -53,7 +53,7 @@ public class NBCLIScenarioParser {
|
||||
.prefix("activities")
|
||||
.prefix(includes)
|
||||
.name(workload)
|
||||
.extension(RawStmtsLoader.YAML_EXTENSIONS)
|
||||
.extension(RawOpsLoader.YAML_EXTENSIONS)
|
||||
.first();
|
||||
return found.isPresent();
|
||||
}
|
||||
@ -67,7 +67,7 @@ public class NBCLIScenarioParser {
|
||||
.prefix("activities")
|
||||
.prefix(includes)
|
||||
.name(workloadName)
|
||||
.extension(RawStmtsLoader.YAML_EXTENSIONS)
|
||||
.extension(RawOpsLoader.YAML_EXTENSIONS)
|
||||
.first();
|
||||
//
|
||||
Content<?> workloadContent = found.orElseThrow();
|
||||
@ -111,10 +111,10 @@ public class NBCLIScenarioParser {
|
||||
.prefix(SEARCH_IN)
|
||||
.prefix(includes)
|
||||
.name(workloadName)
|
||||
.extension(RawStmtsLoader.YAML_EXTENSIONS)
|
||||
.extension(RawOpsLoader.YAML_EXTENSIONS)
|
||||
.first().orElseThrow();
|
||||
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
|
||||
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
|
||||
OpsDocList scenariosYaml = OpsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
|
||||
Scenarios scenarios = scenariosYaml.getDocScenarios();
|
||||
|
||||
String[] nameparts = scenarioName.split("\\.",2);
|
||||
@ -314,12 +314,12 @@ public class NBCLIScenarioParser {
|
||||
}
|
||||
|
||||
Content<?> content = NBIO.all().prefix(SEARCH_IN)
|
||||
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
|
||||
.name(referenced).extension(RawOpsLoader.YAML_EXTENSIONS)
|
||||
.one();
|
||||
|
||||
StmtsDocList stmts = null;
|
||||
OpsDocList stmts = null;
|
||||
try {
|
||||
stmts = StatementsLoader.loadContent(logger, content, Map.of());
|
||||
stmts = OpsLoader.loadContent(content, Map.of());
|
||||
if (stmts.getStmtDocs().size() == 0) {
|
||||
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
|
||||
continue;
|
||||
@ -379,7 +379,7 @@ public class NBCLIScenarioParser {
|
||||
|
||||
List<Content<?>> activities = searchin
|
||||
.prefix(includes)
|
||||
.extension(RawStmtsLoader.YAML_EXTENSIONS)
|
||||
.extension(RawOpsLoader.YAML_EXTENSIONS)
|
||||
.list();
|
||||
|
||||
return filterForScenarios(activities);
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -24,6 +24,22 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
public class GraalJsEvaluatorTest {
|
||||
|
||||
// TODO:
|
||||
|
||||
/**
|
||||
* This evaluator may benefit from being compiled. Presently it give a warning:
|
||||
* <pre>{@code
|
||||
* [To redirect Truffle log output to a file use one of the following options:
|
||||
* * '--log.file=<path>' if the option is passed using a guest language launcher.
|
||||
* * '-Dpolyglot.log.file=<path>' if the option is passed using the host Java launcher.
|
||||
* * Configure logging using the polyglot embedding API.]
|
||||
* [engine] WARNING: The polyglot context is using an implementation that does not support runtime compilation.
|
||||
* The guest application code will therefore be executed in interpreted mode only.
|
||||
* Execution only in interpreted mode will strongly impact the guest application performance.
|
||||
* For more information on using GraalVM see https://www.graalvm.org/java/quickstart/.
|
||||
* To disable this warning the '--engine.WarnInterpreterOnly=false' option or use the '-Dpolyglot.engine.WarnInterpreterOnly=false' system property.
|
||||
* }</pre>
|
||||
*/
|
||||
@Test
|
||||
public void testBasicOperations() {
|
||||
GraalJsEvaluator<Long> ne = new GraalJsEvaluator<>(Long.class);
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -18,9 +18,10 @@ package io.nosqlbench.engine.api.templating;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -34,11 +35,11 @@ public class CommandTemplateTest {
|
||||
|
||||
@Test
|
||||
public void testCommandTemplate() {
|
||||
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
|
||||
"statements:\n" +
|
||||
OpsDocList opsDocs = OpsLoader.loadString("" +
|
||||
"ops:\n" +
|
||||
" - s1: test1=foo test2=bar",
|
||||
Map.of());
|
||||
OpTemplate optpl = stmtsDocs.getStmts().get(0);
|
||||
OpTemplateFormat.yaml, Map.of(), null);
|
||||
OpTemplate optpl = opsDocs.getOps().get(0);
|
||||
CommandTemplate ct = new CommandTemplate(optpl);
|
||||
assertThat(ct.isStatic()).isTrue();
|
||||
}
|
||||
@ -46,14 +47,14 @@ public class CommandTemplateTest {
|
||||
@Test
|
||||
public void testCommandTemplateFormat() {
|
||||
Gson gson = new GsonBuilder().setPrettyPrinting().create();
|
||||
StmtsDocList stmtsDocs = StatementsLoader.loadString("" +
|
||||
"statements:\n" +
|
||||
OpsDocList stmtsDocs = OpsLoader.loadString("" +
|
||||
"ops:\n" +
|
||||
" - s1: test1=foo test2={bar}\n" +
|
||||
" bindings:\n" +
|
||||
" bar: NumberNameToString();\n",
|
||||
Map.of()
|
||||
OpTemplateFormat.yaml, Map.of(), null
|
||||
);
|
||||
OpTemplate optpl = stmtsDocs.getStmts().get(0);
|
||||
OpTemplate optpl = stmtsDocs.getOps().get(0);
|
||||
CommandTemplate ct = new CommandTemplate(optpl);
|
||||
String format = gson.toJson(ct);
|
||||
logger.debug(format);
|
||||
|
@ -51,51 +51,6 @@
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -30,7 +30,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDu
|
||||
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
|
||||
import io.nosqlbench.engine.api.activityapi.input.InputType;
|
||||
import io.nosqlbench.engine.api.activityapi.output.OutputType;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawOpsLoader;
|
||||
import io.nosqlbench.engine.core.annotation.Annotators;
|
||||
import io.nosqlbench.engine.core.lifecycle.process.NBCLIErrorHandler;
|
||||
import io.nosqlbench.engine.core.lifecycle.activity.ActivityTypeLoader;
|
||||
@ -165,7 +165,11 @@ public class NBCLI implements Function<String[], Integer> {
|
||||
|
||||
logger = LogManager.getLogger("NBCLI");
|
||||
loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
|
||||
logger.info(() -> "Configured scenario log at " + loggerConfig.getLogfileLocation());
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info(() -> "Configured scenario log at " + loggerConfig.getLogfileLocation());
|
||||
} else {
|
||||
System.err.println("Configured scenario log at " + loggerConfig.getLogfileLocation());
|
||||
}
|
||||
logger.debug("Scenario log started");
|
||||
|
||||
// Global only processing
|
||||
@ -304,7 +308,7 @@ public class NBCLI implements Function<String[], Integer> {
|
||||
Optional<Content<?>> tocopy = NBIO.classpath()
|
||||
.prefix("activities")
|
||||
.prefix(options.wantsIncludes())
|
||||
.name(resourceToCopy).extension(RawStmtsLoader.YAML_EXTENSIONS).first();
|
||||
.name(resourceToCopy).extension(RawOpsLoader.YAML_EXTENSIONS).first();
|
||||
|
||||
if (tocopy.isEmpty()) {
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,9 +16,9 @@
|
||||
|
||||
package io.nosqlbench.engine.cli;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
@ -33,13 +33,13 @@ public class NBCLIScenarioParserTemplateVarTest {
|
||||
List<Cmd> cmds = opts.getCommands();
|
||||
cmds.forEach(System.out::println);
|
||||
|
||||
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
|
||||
OpTemplate optpl1 = workload1.getStmts().get(0);
|
||||
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
|
||||
OpTemplate optpl1 = workload1.getOps().get(0);
|
||||
System.out.println("op from cmd1:"+optpl1);
|
||||
assertThat(optpl1.getStmt()).contains("cycle {cycle} replaced replaced\n");
|
||||
|
||||
StmtsDocList workload2 = StatementsLoader.loadPath(null, cmds.get(1).getArg("workload"),cmds.get(1).getParams());
|
||||
OpTemplate optpl2 = workload2.getStmts().get(0);
|
||||
OpsDocList workload2 = OpsLoader.loadPath(cmds.get(1).getArg("workload"),cmds.get(1).getParams());
|
||||
OpTemplate optpl2 = workload2.getOps().get(0);
|
||||
System.out.println("op from cmd2:"+optpl2);
|
||||
assertThat(optpl2.getStmt()).contains("cycle {cycle} def1 def1\n");
|
||||
}
|
||||
@ -50,8 +50,8 @@ public class NBCLIScenarioParserTemplateVarTest {
|
||||
List<Cmd> cmds = opts.getCommands();
|
||||
cmds.forEach(System.out::println);
|
||||
|
||||
StmtsDocList workload1 = StatementsLoader.loadPath(null, cmds.get(0).getArg("workload"),cmds.get(0).getParams());
|
||||
OpTemplate optpl1 = workload1.getStmts().get(0);
|
||||
OpsDocList workload1 = OpsLoader.loadPath(cmds.get(0).getArg("workload"),cmds.get(0).getParams());
|
||||
OpTemplate optpl1 = workload1.getOps().get(0);
|
||||
System.out.println("op from cmd1:"+optpl1);
|
||||
assertThat(optpl1.getStmt()).contains("cycle {cycle} overridden overridden\n");
|
||||
}
|
||||
|
@ -58,51 +58,6 @@
|
||||
</includes>
|
||||
</testResource>
|
||||
</testResources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
|
@ -93,49 +93,5 @@
|
||||
<filtering>true</filtering>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
@ -299,7 +299,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
|
||||
break;
|
||||
case Finished:
|
||||
case Stopping:
|
||||
throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
|
||||
case Errored:
|
||||
break;
|
||||
// throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
|
||||
|
||||
default:
|
||||
throw new RuntimeException("Unmatched run state:" + activity.getRunState());
|
||||
@ -314,16 +316,16 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
|
||||
case Running:
|
||||
tally.awaitNoneOther(RunState.Running, RunState.Finished);
|
||||
break;
|
||||
case Errored:
|
||||
case Stopping:
|
||||
case Stopped:
|
||||
tally.awaitNoneOther(RunState.Stopped, RunState.Finished);
|
||||
tally.awaitNoneOther(RunState.Stopped, RunState.Finished, RunState.Errored);
|
||||
break;
|
||||
case Uninitialized:
|
||||
break;
|
||||
case Finished:
|
||||
tally.awaitNoneOther(RunState.Finished);
|
||||
break;
|
||||
case Stopping:
|
||||
throw new RuntimeException("Invalid requested state in activity executor:" + activity.getRunState());
|
||||
default:
|
||||
throw new RuntimeException("Unmatched run state:" + activity.getRunState());
|
||||
}
|
||||
@ -417,7 +419,7 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
|
||||
}
|
||||
}
|
||||
|
||||
public void startActivity() {
|
||||
public synchronized void startActivity() {
|
||||
// we need an executor service to run motor threads on
|
||||
startMotorExecutorService();
|
||||
startRunningActivityThreads();
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -277,8 +277,10 @@ public class Scenario implements Callable<ExecutionMetricsResult> {
|
||||
}
|
||||
|
||||
Runtime.getRuntime().removeShutdownHook(scenarioShutdownHook);
|
||||
scenarioShutdownHook.run();
|
||||
var runHook = scenarioShutdownHook;
|
||||
scenarioShutdownHook = null;
|
||||
runHook.run();
|
||||
logger.debug("removing scenario shutdown hook");
|
||||
}
|
||||
|
||||
public void notifyException(Thread t, Throwable e) {
|
||||
@ -340,7 +342,7 @@ public class Scenario implements Callable<ExecutionMetricsResult> {
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void finish() {
|
||||
public void finish() {
|
||||
logger.debug("finishing scenario");
|
||||
endedAtMillis = System.currentTimeMillis(); //TODO: Make only one endedAtMillis assignment
|
||||
if (this.state == State.Running) {
|
||||
|
@ -76,53 +76,4 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -66,49 +66,6 @@
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
|
@ -45,7 +45,6 @@
|
||||
|
||||
|
||||
<build>
|
||||
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
@ -54,71 +53,6 @@
|
||||
</includes>
|
||||
</resource>
|
||||
</resources>
|
||||
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
|
||||
<!-- <plugin>-->
|
||||
<!-- <groupId>org.apache.maven.plugins</groupId>-->
|
||||
<!-- <artifactId>maven-javadoc-plugin</artifactId>-->
|
||||
<!-- <version>3.1.1</version>-->
|
||||
<!-- <configuration>-->
|
||||
<!-- <doctitle>nosqlbench Extensions</doctitle>-->
|
||||
<!-- <windowtitle>nosqlbench Extensions</windowtitle>-->
|
||||
<!-- </configuration>-->
|
||||
<!-- <executions>-->
|
||||
<!-- <execution>-->
|
||||
<!-- <id>attach-javadoc</id>-->
|
||||
<!-- <goals>-->
|
||||
<!-- <goal>jar</goal>-->
|
||||
<!-- </goals>-->
|
||||
<!-- </execution>-->
|
||||
<!-- </executions>-->
|
||||
<!-- </plugin>-->
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
@ -56,52 +56,4 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2022 nosqlbench
|
||||
* Copyright (c) 2022-2023 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.engine.rest.services;
|
||||
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
|
||||
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
|
||||
import io.nosqlbench.engine.api.scenarios.WorkloadDesc;
|
||||
import io.nosqlbench.engine.rest.transfertypes.WorkspaceItemView;
|
||||
|
@ -14,7 +14,9 @@
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>io.nosqlbench</groupId>
|
||||
@ -34,8 +36,9 @@
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
|
||||
<javadoc.name>nosqlbench</javadoc.name>
|
||||
|
||||
<jacoco.version>0.8.8</jacoco.version>
|
||||
<maven.compiler.source>17</maven.compiler.source>
|
||||
<maven.compiler.target>17</maven.compiler.target>
|
||||
|
||||
</properties>
|
||||
|
||||
@ -142,6 +145,32 @@
|
||||
<version>1.10.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math4-core</artifactId>
|
||||
<version>4.0-beta1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math4-legacy</artifactId>
|
||||
<version>4.0-beta1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math4-legacy-core</artifactId>
|
||||
<version>4.0-beta1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math4-legacy-exception</artifactId>
|
||||
<version>4.0-beta1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-statistics-distribution</artifactId>
|
||||
<version>1.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.openjdk.jmh</groupId>
|
||||
<artifactId>jmh-core</artifactId>
|
||||
@ -270,7 +299,7 @@
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
<version>1.12.382</version>
|
||||
<version>1.12.393</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@ -303,12 +332,6 @@
|
||||
<version>2.12.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<version>3.6.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-csv</artifactId>
|
||||
@ -472,6 +495,7 @@
|
||||
<version>1.7.1</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
@ -510,7 +534,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.9.0</version>
|
||||
<version>3.10.1</version>
|
||||
<configuration>
|
||||
<debug>true</debug>
|
||||
<target>17</target>
|
||||
@ -528,8 +552,10 @@
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.0.0-M6</version>
|
||||
<configuration>
|
||||
<argLine>-ea ${argLine}</argLine>
|
||||
<systemPropertyVariables>
|
||||
<Log4jContextSelector>org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
|
||||
<Log4jContextSelector>
|
||||
org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
|
||||
</Log4jContextSelector>
|
||||
</systemPropertyVariables>
|
||||
<parallel>methods</parallel>
|
||||
@ -546,6 +572,57 @@
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<phase>process-test-classes</phase>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<destFile>${project.build.directory}/jacoco-data.exec</destFile>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<dataFile>${project.build.directory}/jacoco-data.exec</dataFile>
|
||||
<outputDirectory>${project.build.directory}/jacoco-reports</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
@ -706,7 +783,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<version>3.10.1</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
|
@ -45,52 +45,4 @@
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
</project>
|
||||
|
@ -150,52 +150,6 @@
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>jacoco-check</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>check</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<rule>
|
||||
<element>BUNDLE</element>
|
||||
<limits>
|
||||
<limit>
|
||||
<counter>INSTRUCTION</counter>
|
||||
<value>COVEREDRATIO</value>
|
||||
<minimum>0.00</minimum>
|
||||
<maximum>1.00</maximum>
|
||||
</limit>
|
||||
</limits>
|
||||
</rule>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user