IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:nosqlbench/nosqlbench into adapter-cockroachdb

Browse Source
This commit is contained in:
Madhavan Sridharan
2023-01-14 10:17:05 -05:00
parent 0f7e9eda9a a49a9e9380
commit 582cbddb26
114 changed files with 3793 additions and 1530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
.github/workflows/build.yml vendored
View File

@@ -25,17 +25,24 @@ jobs:
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
# - name: mvn-package
# run: mvn package
- name: mvn verify
run: mvn verify
- name: Capture
- name: Installing dependencies
run: mvn clean install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
- name: Uploading [nosqlbench] test coverage
uses: actions/upload-artifact@v3
with:
name: codecov-report
path: codecov-report.tar
- name: Collecting [nosqlbench] logfiles
if: success() || failure()
run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
- name: Archive Test Results
- name: Uploading test result log files
if: success() || failure()
uses: actions/upload-artifact@v3
with:
@@ -43,34 +50,32 @@ jobs:
path: logfiles.tar
- name: export docs
if: success()
run: nb5/target/nb5 export-docs
- name: upload docs artifact
if: success()
uses: actions/upload-artifact@v3
with:
name: exported-docs
path: exported_docs.zip
docs:
needs: build
runs-on: ubuntu-20.04
steps:
- name: set git username
run: git config --global user.email "${{ secrets.NBDROID_EMAIL }}"
- name: set git email
run: git config --global user.name "${{ secrets.NBDROID_NAME }}"
- name: download exported-docs
- name: download exported_docs
uses: actions/download-artifact@v3
with:
name: exported-docs
- name: unzip docs
run: unzip exported_docs.zip
- run: ls -la
- name: clone nosqlbench-build-docs
@@ -85,18 +90,26 @@ jobs:
git remote set-url origin https://${{secrets.NBDROID_NAME}}:${{secrets.NBDROID_TOKEN}}@github.com/nosqlbench/nosqlbench-build-docs.git
git remote -v
- name: import zipfile
run: |
mv exported_docs.zip nosqlbench-build-docs/
- name: set CNAME
run: |
echo "builddocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME
- name: commit changes
run: |
cd nosqlbench-build-docs
git add exported_docs.zip
- name: push changes
env:
NBDROID_NAME: ${{ secrets.NBDROID_NAME }}
NBDROID_TOKEN: ${{ secrets.NBDROID_TOKEN }}
run: |
set -x
find . -ls
rsync -av --delete -I --exclude '_index.md' drivers/ nosqlbench-build-docs/site/content/docs/drivers
rsync -av --delete -I --exclude '_index.md' bindings/ nosqlbench-build-docs/site/content/docs/bindings
echo "previewdocs.nosqlbench.io" > nosqlbench-build-docs/site/static/CNAME
cd nosqlbench-build-docs
git add -A
CHANGES=$(git status --porcelain 2>/dev/null| wc -l)
echo "found $CHANGES to push for doc updates"
if (( $CHANGES > 0 ))

63
.github/workflows/pr-testing-cov.yml vendored
View File

@@ -1,63 +0,0 @@
name: nb-pr-test-cov
on:
push:
branches:
- jeffb/cc-test
# pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
name: checkout nosqlbench
- uses: actions/setup-java@v3
name: setup java
with:
java-version: '17'
java-package: jdk
architecture: x64
distribution: 'temurin'
- name: Cache Maven packages
uses: actions/cache@v1
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
- name: mvn verify
run: mvn verify
- name: Capture
if: success() || failure()
run: tar -cvf logfiles.tar [a-zA-Z]**/logs/*
- name: Archive Test Results
if: success() || failure()
uses: actions/upload-artifact@v3
with:
name: test-results
path: logfiles.tar
- name: Upload to Codecov
uses: codecov/codecov-action@v3
- name: package surefire artifacts
if: failure()
run: |
rm -rf artifacts
mkdir artifacts
find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
zip -r artifacts.zip artifacts
- uses: actions/upload-artifact@master
name: upload surefire-artifacts
if: failure()
with:
name: surefire-artifacts
path: artifacts.zip

2
.gitignore vendored
View File

@@ -1,3 +1,4 @@
exported_docs.zip
.nosqlbench/**
.run/**
workspaces/**
@@ -17,4 +18,5 @@ bin/**
**/generated
**/.DS_Store
**/generated/**
**/.flattened-pom.xml
/deptree.txt

214
RELEASENOTES.md
View File

@@ -1,185 +1,29 @@
- 808399ea1 (HEAD -> main, origin/main) github actions release testing
- a93c5acac checkpoint release for nb5
- b8e0bcd26 Merge branch 'main' of github.com:nosqlbench/nosqlbench
- 0eb71686a removed stale modules
- d785fe4bb Merge pull request #893 from nosqlbench/my-nosqlbench-440
- ea93403f4 (origin/my-nosqlbench-440) removed suffix parameter from tcp driver op dispensers (We have string templates to do that now)
- bca21e86b codecov impl
- 11e4c8cc4 Scan fix: cs #17, implicit narrowing
- 6881c0790 Revert "nosqlbench-440 convert tcpclient driver from ActivityType to DriverAdapter e:1d"
- f8eecd541 Scan fix: cs #35, implicit narrowing
- 1b2655c27 nosqlbench-440 convert tcpclient driver from ActivityType to DriverAdapter e:1d
- 376a9b04a combine build phases for main verification
- 4c2a18148 run example tests serially for now
- 4e418c7c7 remove engine-clients module
- 0b7d0aaad remove engine-clients from dependencies
- 1af40544b allow build on push for this branch
- f4aa80f9e remove previous "nb" module, which will be replaced by the nb5 module soon
- ccb2aefec update module versions
- edbf25abe remove nb module profile
- 8323c78e4 update metrics version to drop old slf4j dependency
- 1e7d67697 deactivate rest module with old dependencies
- ac279d364 consolidate error tests
- 0bc4064f3 sync stale versions on inactive modules
- 9f1c20e10 update and fill-in missing slf4j stub layer
- 9c73c4e18 remove previous "nb" module, which will be replaced by the nb5 module soon
- 2d16ff3b5 Revert "NB5 address high priority code scans" (#895)
- ecfe3d633 Narrowing fixes
- f1bb2ed06 use alternate inline binding form for zola
- 77243f325 make anchor links zola compatible
- b334a514f remove duplicitous doc file
- dfe1c0775 Merge pull request #896 from nosqlbench/nosqlbench-886-noslf4j
- 1c86fac3f (origin/nosqlbench-886-noslf4j, nosqlbench-886-noslf4j) combine build phases for main verification
- 688a6f728 run example tests serially for now
- c86029818 remove engine-clients module
- 3e2802a6e remove engine-clients from dependencies
- 50207b38f allow build on push for this branch
- 865c0ac38 remove previous "nb" module, which will be replaced by the nb5 module soon
- 8cb4f7b77 update module versions
- 55c6bca95 remove nb module profile
- f75eee5ac update metrics version to drop old slf4j dependency
- f379ad292 deactivate rest module with old dependencies
- f41250569 consolidate error tests
- fb8e27d33 sync stale versions on inactive modules
- 36fa9461d update and fill-in missing slf4j stub layer
- f38a635d7 Modified HDR Digits to 3 to save space on MetricsIntegrationTest.
- ffcc29414 Cleaned up "filename" configuration option from TCP. (Only available for Stdout)
- 45d00b187 Revert "NB5 address high priority code scans" (#895)
- 6146d186d Modified Writer Adapter Behavior for TCPServer to use poll and offer with a specified retry time (1s)
- 89c3bb453 Merge pull request #894 from nosqlbench/jeffb/code-scans-high
- d46ae1f12 Merge pull request #885 from nosqlbench/docfixes
- ccc7f7ab3 Cleaned up code after review.
- 23b79628b (origin/jeffb/code-scans-high) Narrowing fixes
- bdf61845f nosqlbench-440 convert tcpclient driver from ActivityType to DriverAdapter e:1d
- 0139ea0ad fix: upgrade org.eclipse.jetty:jetty-servlets from 11.0.12 to 11.0.13 (#892)
- 277093d30 fix: upgrade org.eclipse.jetty:jetty-server from 11.0.12 to 11.0.13 (#891)
- f5ed9b011 fix: upgrade org.eclipse.jetty:jetty-servlet from 11.0.12 to 11.0.13 (#890)
- 9d577e669 fix: upgrade org.eclipse.jetty:jetty-rewrite from 11.0.12 to 11.0.13 (#889)
- 3fc6b195a fix: adapter-dynamodb/pom.xml to reduce vulnerabilities (#888)
- 19b8743f7 fix: upgrade com.github.oshi:oshi-core-java11 from 6.3.2 to 6.4.0 (#887)
- 38088659c remove previous "nb" module, which will be replaced by the nb5 module soon
- 7bcc7d6bf fix: upgrade joda-time:joda-time from 2.12.1 to 2.12.2 (#884)
- e4fd27649 (origin/docfixes, docfixes) use alternate inline binding form for zola
- 08fb6ca90 make anchor links zola compatible
- 972a6af65 remove duplicitous doc file
- 704e2751d Merge remote-tracking branch 'origin/main'
- c316bed26 support alternate inline format to work around docs shortcodes
- 52d5d10b8 Merge pull request #878 from nosqlbench/nosqlbench-875-categories
- dd1fc35d3 Merge pull request #880 from nosqlbench/nosqlbench-797-callable
- b6caecba4 Merge pull request #883 from nosqlbench/nosqlbench-881-fixtest
- 6731937c9 (nosqlbench-881-fixtest) modify logger calls for computable messages via closures
- a06cf565e enable async logging for tests
- 1af99a9b4 make logging-sensitive tests work with async logging
- 48055fe95 reduce logging output for integrated tests
- fd8e871da use local logger at debug level for any test output
- 78afbfbc5 Merge branch 'main' of github.com:nosqlbench/nosqlbench
- 10080720a (nosqlbench-797-callable) PR fixes from comments
- f8f7ca052 Merge pull request #872 from nosqlbench/nosqlbench-797-callable
- 54546ce9a (nosqlbench-875-categories) put required categories on all binding functions, not just one of each name
- b528a4e29 annotation processor asserts presence of categories on binding functions
- 74c234ddc Merge branch 'main' of github.com:nosqlbench/nosqlbench
- 301f1e2d2 added missing category for TriangularStepFunction
- 068c758bb Merge pull request #873 from nosqlbench/snyk-upgrade-e2924aae5a1f341565ee282682130a8f
- 77f31d193 fix: upgrade io.swagger.core.v3:swagger-models from 2.2.6 to 2.2.7
- 6da3798bb Update build.yml
- 1e0b18968 Merge pull request #871 from nosqlbench/snyk-upgrade-5b9126aa6ad92dc3e050d3528eccae75
- 6c8782842 change tests System.out calls to use logger directly
- 4f4b4982c templatize test logging level for logger setup and root logger
- d6c9f5153 test cleanup
- a6ff30d36 fix: upgrade io.dropwizard.metrics:metrics-graphite from 4.2.12 to 4.2.13
- d80632773 merge fixes for: improve concurrency patterns for activity execution
- b9365bff7 improve concurrency patterns for activity execution
- bf5a31b34 implement efficient concurrent thread state signaling
- 0de80887b move log4j.xml to log4j2.xml
- 077b41a3c misc naming, typo and formatting improvements
- 1816381f9 make integrated tests more robust
- 1416c71d9 internalize activity start logic
- c076b1079 make http extension docs distinct from http adapter docs
- 31dc3ce8f package housekeeping
- a1cde761f fix unapplied bug in diag space
- 3f14fefcb delegate activity instancing to dedicated loader
- ba42bfea4 allow conversion of missing config types
- 132192322 remove unused code paths to simplify refactoring
- bd129c442 run activities within a dedicated executor
- 09f776ad2 Merge pull request #870 from nosqlbench/stargate-v2-update
- 72dd19cb2 fixing operation syntax
- 2a62cc882 fix: upgrade com.github.docker-java:docker-java-transport-okhttp from 3.2.13 to 3.2.14 (#868)
- 7bc4265af fix: adapter-dynamodb/pom.xml to reduce vulnerabilities (#867)
- c4415e6c0 Merge pull request #863 from nosqlbench/my-NB5-I446
- 8fd13fcdd (origin/my-NB5-I446) added explicit cast to explicitly narrow precision for security purposes
- dc2174b95 fixed missing licenses
- e18dcbc22 Merge pull request #864 from nosqlbench/nosqlbench-861-dryrun
- fc7d72fac Merge pull request #866 from nosqlbench/nosqlbench-865-urlencode
- 3013dc315 fixed s4j.md to be compatible with export-docs
- 74bfbc691 Merge branch 'main' into my-NB5-I446
- 084dddf1d nosqlbench-865 Apply urlencoder logic in http for all op field forms
- d1d21c8e5 nosqlbench-861 Allow dry-run of op generation.
- 39a9f5e92 further cleanup of unused comments and commands
- b63eba35c fixed pulsar.md for export-docs after merge
- a06e1556f merged my-NB5-I446 with main
- f76c63b99 only launch build workflow when pushing to main branch
- 7eeb1d288 fixed unused dependency failure
- ad855e625 modified build workflow v2
- 99a4c9fb4 modified build workflow
- ec8ed024f Changed build workflow to re-add verify and run on non-release builds, removed unused dependency from nbr
- c149684d0 moved from commonmark to inhouse MutableMarkdown for verification
- bb28f0374 fix: mvn-defaults/pom.xml to reduce vulnerabilities (#859)
- 3891073ff fix: upgrade io.swagger.parser.v3:swagger-parser from 2.1.7 to 2.1.9 (#858)
- 3cd978950 fix: upgrade com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider from 2.14.0 to 2.14.1 (#857)
- 97bb2e7b9 fix: upgrade org.openjdk.jmh:jmh-core from 1.35 to 1.36 (#852)
- 60252ed2b fix: upgrade io.dropwizard.metrics:metrics-core from 4.2.12 to 4.2.13 (#851)
- 8545673f7 fix: upgrade com.github.oshi:oshi-core-java11 from 6.3.1 to 6.3.2 (#850)
- 7ba2a3de3 Moved CNAME to correct directory
- 04153d729 simplified staging command
- e29bfddc1 (origin/timeout_fix) avoid JVM timeout bug with lower default
- 9f3cae591 fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.330 to 1.12.347 (#849)
- 42d1403fb fix: nb-api/pom.xml to reduce vulnerabilities (#848)
- c514a62fa fix: upgrade org.openjdk.jmh:jmh-generator-annprocess from 1.35 to 1.36 (#853)
- 0c0156201 fix: adapter-dynamodb/pom.xml to reduce vulnerabilities (#855)
- bca55d594 Added status command for debug
- b1f0d590f Added hard overwrite for existing files
- d472b071d excluded _index.md from rsync dynamic update
- f0507cafb Merge pull request #854 from yabinmeng/main
- 37f889359 Fix Kafka producer transaction issue and add support for subscribing from multiple topics. NB yaml files and README update
- 3943a2945 Merge pull request #846 from nosqlbench/dependabot/maven/mvn-defaults/io.netty-netty-codec-haproxy-4.1.86.Final
- 17e4cb323 Merge pull request #847 from yabinmeng/main
- 1850a7c16 Add NB5 Kafka driver
- 1adb553b3 build(deps): bump netty-codec-haproxy in /mvn-defaults
- 2b39f3c35 Merge branch 'main' into nosqlbench-797-callable
- 21a33aec9 final version v1
- 5ccb06470 back to base simple case
- ee71bdd17 added checkout action v5
- 0c8ac02f3 added checkout action v4
- 2c61e9c52 added checkout action v3
- 875d0d960 Merge pull request #844 from nosqlbench/nosqlbench-843-cqlgen-mv
- 4383ba208 added checkout action v2
- 26e0b4cb0 added checkout action
- 4a178fc21 added git remote command to debug remote v2
- 43fa3e725 added git remote command to debug remote
- c0117351f Merge pull request #845 from nosqlbench/snyk-fix-b8424b271d02d50c5baa47f9e8745c03
- df534fd83 Merge branch 'main' into snyk-fix-b8424b271d02d50c5baa47f9e8745c03
- 5a47fded6 corrected workflow file for debug v2
- e09a93ea9 corrected workflow file for debug
- 0e9dff474 testing simple case for access debug
- f660c6272 converted to simple case for access debug
- 35f4bb982 (nosqlbench-843-cqlgen-mv) avoid NPE when cqlgen parses materialized views
- 21eb031da Merge pull request #837 from nosqlbench/nosqlbench-835-count-warning
- 8240844f6 Merge pull request #838 from nosqlbench/nosqlbench-836
- b3bcc3ce5 Merge pull request #839 from nosqlbench/nosqlbench-834-triangularstep
- 21bc389f2 Merge pull request #842 from nosqlbench/nosqlbench-841-hashedrange
- 8a48a020d (nosqlbench-841-hashedrange) allow zero width ranges
- 862058d80 (nosqlbench-834-triangularstep, nosqlbench-797) make TriangularStep ctor public
- 9d1e5c14d (nosqlbench-836) nosqlbench-836 expand default data retention in promql from 6 mo to 10 years
- 4b04cb3aa (nosqlbench-835-count-warning) move warning to ctor
- 457c75bc3 (nosqlbench-841-hashedrangesize) Merge pull request #832 from yabinmeng/main
- 2f5dd1f74 clean up
- bbbdb60dc added change directory before git command
- b9efec960 Modified git commands in build workflow for docs
- ae1a3af9c Corrected build workflow clone path
- ea46cbb74 removed debug commands and corrected build workflow clone path
- b77940715 Added debug commands for GHA
- 10d9ce314 Disabled verify for testing
- e7705d13d NB5-I446 Modified export-docs command to export organized driver and binding docs, also removed gendocs command (VirtDataGenDocsApp now used for generating bindings in StringBuilders to pass to the zipstream) added the export-docs build workflow job Updated build.yml to fix path to nb5 in workflow uploading/downloading docs artifacts in build workflow Attempt to correct the path to the exported docs to upload them Corrected exported-docs path Corrected yml build workflow error corrected naming of exported docs
- c08f8b6b3 1) Add NB5 S4J example yaml files 2) Update NB5 S4J readme file 3) Minor code adjustments
- 9a00af080 Merge branch 'nosqlbench:main' into main
- c102b213a github actions release test
- e04eef814 (HEAD -> main) temporary work-around for release plugin incompatibility
- 2ce8a0e6a (origin/main) release candidate
- ff188d486 actions-for-docs (#919)
- 97e992bce version updates and pom structure improvements (#918)
- 4f6bea91d switch over to snakeyaml-engine
- d06e01c22 (origin/nosqlbench-916-snakeyaml, nosqlbench-916-snakeyaml) Merge pull request #915 from nosqlbench/my-cql-astra-example
- 459777a94 Merge pull request #914 from nosqlbench/nosqlbench-cvefixes
- e2ad639b7 (origin/my-cql-astra-example) Added cql Astra workload yaml example for docs, also fixed regex security issue by removing CompatibilityFixups
- 2265f1e58 (origin/nosqlbench-cvefixes, nosqlbench-cvefixes) remove endpoints which are no longer needed
- 740faffda typo fixes
- 31478ae59 Nosqlbench 904 version5 (#908)
- 9c25ad933 Code coverage integration (#897)
- 3258e4e06 Merge pull request #912 from nosqlbench/my-security-regex
- b6ab7293e newline fix
- 4484292cb md5 deprecation (#909)
- 9767e98bc limited name length to limit user input causing slow regex run, Removed starting boundary delimiter v2
- 78ddf0597 limited name length to limit user input causing slow regex run, Removed starting boundary delimiter
- 7b6decb90 corrected non-escaped minus sign in regex, and limited delimeter characters to a maximum of 2 to account for comma-space separated values
- 7f446145b Provided limited number of newlines in SSL KEY and CERT Patterns in Regex, Named the flag for more clarity
- a95acae20 Merge pull request #907 from yabinmeng/main
- 2aeeea125 Separate ActivityExecutor.forceStop() method out from the stop() method. Removing the "forcing" flag in the stop() method.
- 88fa61a49 Merge pull request #905 from nosqlbench/snyk-fix-cfc004029344dd452bce66ee2666e50c
- c172a140e Add 'forceStop' NB CLI option in NB5 (port over from similar functionally in NB4 - PR# 675)
- f77552b8f fix: nb-spectest/pom.xml to reduce vulnerabilities
- 7a1400c51 Merge pull request #899 from nosqlbench/snyk-upgrade-cd0da6fb32f57e40946c06204916a1e7
- 949a458f3 Merge pull request #900 from nosqlbench/docs_workflow
- bbcf493e9 (origin/docs_workflow, docs_workflow) fixed artifact name
- 80aed220a (origin/snyk-upgrade-cd0da6fb32f57e40946c06204916a1e7) fix: upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.364 to 1.12.368
- a1cd44cb1 remove previous "nb" module, which will be replaced by the nb5 module soon

52
adapter-cqld4/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -38,7 +38,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -50,12 +50,12 @@
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>3.4.13</version>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>tinkergraph-gremlin</artifactId>
<version>3.4.13</version>
<version>3.6.1</version>
</dependency>
<dependency>
@@ -140,6 +140,48 @@
<followSymLinks>false</followSymLinks>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>

2
adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzer.java
View File

@@ -16,7 +16,7 @@
package io.nosqlbench.analysis;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java
View File

@@ -20,7 +20,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
import io.nosqlbench.cqlgen.api.BindingsLibrary;

77
adapter-cqld4/src/main/resources/activities/baselines/cql-keyvalue-astra.md Normal file
View File

@@ -0,0 +1,77 @@
---
title: CQL Key-Value
weight: 1
---
## Description
The CQL Key-Value workload demonstrates the simplest possible schema with payload data. This is useful for measuring
system capacity most directly in terms of raw operations. As a reference point, provides some insight around types of
workloads that are constrained around messaging, threading, and tasking, rather than bulk throughput.
During preload, all keys are set with a value. During the main phase of the workload, random keys from the known
population are replaced with new values which never repeat. During the main phase, random partitions are selected for
upsert, with row values never repeating.
## Operations
### insert (rampup, main)
insert into baselines.keyvalue (key, value) values (?,?);
### read (main)
select * from baselines.keyvalue where key=?key;
## Data Set
### baselines.keyvalue insert (rampup)
- key - text, number as string, selected sequentially up to keycount
- value - text, number as string, selected sequentially up to valuecount
### baselines.keyvalue insert (main)
- key - text, number as string, selected uniformly within keycount
- value - text, number as string, selected uniformly within valuecount
### baselines.keyvalue read (main)
- key - text, number as string, selected uniformly within keycount
## Workload Parameters
This workload has no adjustable parameters when used in the baseline tests.
When used for additional testing, the following parameters should be supported:
- keycount - the number of unique keys
- valuecount - the number of unique values
## Key Performance Metrics
Client side metrics are a more accurate measure of the system behavior from a user's perspective. For microbench and
baseline tests, these are the only required metrics. When gathering metrics from multiple server nodes, they should be
kept in aggregate form, for min, max, and average for each time interval in monitoring. For example, the avg p99 latency
for reads should be kept, as well as the min p99 latency for reads. If possible metrics, should be kept in plot form,
with discrete histogram values per interval.
### Client-Side
- read ops/s
- write ops/s
- read latency histograms
- write latency histograms
- exception counts
### Server-Side
- pending compactions
- bytes compacted
- active data on disk
- total data on disk
# Notes on Interpretation
Once the average ratio of overwrites starts to balance with the rate of compaction, a steady state should be achieved.
At this point, pending compactions and bytes compacted should be mostly flat over time.

96
adapter-cqld4/src/main/resources/activities/baselines/cql-keyvalue-astra.yaml Normal file
View File

@@ -0,0 +1,96 @@
description: A workload with only text keys and text values
scenarios:
default:
schema: run driver=cql tags==phase:schema threads==1 cycles==UNDEF
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
astra:
schema: run driver=cql tags==phase:schema-astra threads==1 cycles==UNDEF
rampup: run driver=cql tags==phase:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=cql tags==phase:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:
seq_key: Mod(<<keycount:1000000000>>); ToString() -> String
seq_value: Hash(); Mod(<<valuecount:1000000000>>); ToString() -> String
rw_key: <<keydist:Uniform(0,1000000000)->int>>; ToString() -> String
rw_value: Hash(); <<valdist:Uniform(0,1000000000)->int>>; ToString() -> String
blocks:
- name: schema
tags:
phase: schema
params:
prepared: false
statements:
- create-table: |
create table if not exists <<keyspace:baselines>>.<<table:keyvalue>> (
key text,
value text,
PRIMARY KEY (key)
);
tags:
name: create-table
- name: schema-astra
tags:
phase: schema-astra
params:
prepared: false
statements:
- create-table: |
create table if not exists <<keyspace:baselines>>.<<table:keyvalue>> (
key text,
value text,
PRIMARY KEY (key)
);
tags:
name: create-table-astra
- name: rampup
tags:
phase: rampup
params:
cl: <<write_cl:LOCAL_QUORUM>>
statements:
- rampup-insert: |
insert into <<keyspace:baselines>>.<<table:keyvalue>>
(key, value)
values ({seq_key},{seq_value});
tags:
name: rampup-insert
- name: verify
tags:
phase: verify
type: read
params:
cl: <<read_cl:LOCAL_QUORUM>>
statements:
- verify-select: |
select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={seq_key};
verify-fields: key->seq_key, value->seq_value
tags:
name: verify
- name: main-read
tags:
phase: main
type: read
params:
ratio: 5
cl: <<read_cl:LOCAL_QUORUM>>
statements:
- main-select: |
select * from <<keyspace:baselines>>.<<table:keyvalue>> where key={rw_key};
tags:
name: main-select
- name: main-write
tags:
phase: main
type: write
params:
ratio: 5
cl: <<write_cl:LOCAL_QUORUM>>
statements:
- main-insert: |
insert into <<keyspace:baselines>>.<<table:keyvalue>>
(key, value) values ({rw_key}, {rw_value});
tags:
name: main-insert

100
adapter-cqld4/src/main/resources/cqld4.md
View File

@@ -1,6 +1,6 @@
# cqld4 driver
# cqld4
This is the newly revamped (beta) driver for CQL which uses the DataStax OSS Driver version 4. As
This is the newly revamped driver for CQL which uses the DataStax OSS Driver version 4. As
there was a significant restructuring of the APIs between CQL driver 4.x and previous versions, this
driver is a clean and separate implementation which aims to use the features of version 4.x of the
native driver directly as well as new internal NoSQLBench APIs.
@@ -38,20 +38,20 @@ mixing in JSON or remote URLs.
Configure directly from a config file, or classpath resource:
# If this isn't found in the file system, the classpath will also be checked.
driverconfig=myconfig.json
```shell
# If this isn't found in the file system, the classpath will also be checked.
nb5 ... driverconfig=myconfig.json
```
Configure directly from JSON:
driverconfig='{basic.request.timeout:"2 seconds"}'
```shell
nb5 ... driverconfig='{basic.request.timeout:"2 seconds"}'
```
Configure directly form a remote URL:
driverconfig='http://gist.github.com...'
Configure from multiple sources:
driverconfig=myconfig.json
```shell
nb5 ... driverconfig='http://gist.github.com...'
```
### Basic Cqld4 driver options
@@ -109,48 +109,50 @@ classic form have not changed.
## CQLd4 Op Template Examples
ops:
```yaml
ops:
# prepared statement
# allows for parameterization via bindings, and uses prepared statements internally
example-prepared-cql-stmt:
prepared: |
select one, two from buckle.myshoe where ...
# prepared statement
# allows for parameterization via bindings, and uses prepared statements internally
example-prepared-cql-stmt:
prepared: |
select one, two from buckle.myshoe where ...
# prepared statement (verbose form)
example-prepared-cql-stmt-verbose:
type: prepared
stmt: |
select one, two from buckle.myshoe where ...
# prepared statement (verbose form)
example-prepared-cql-stmt-verbose:
type: prepared
stmt: |
select one, two from buckle.myshoe where ...
# simple statement
# allows for parameterization via bindings, but does not use prepared statements internally
example-simple-cql-stmt:
simple: |
select three, four from knock.onthedoor where ...
# simple statement
# allows for parameterization via bindings, but does not use prepared statements internally
example-simple-cql-stmt:
simple: |
select three, four from knock.onthedoor where ...
# raw statement
# pre-renders the statement into a string, with no driver-supervised parameterization
# useful for testing variant DDL where some fields are not parameterizable
# NOTE: the raw form does its best to quote non-literals where needed, but you may
# have to inject single or double quotes in special cases.
example-raw-cql-stmt:
raw: |
create table if not exist {ksname}.{tblname} ...
# raw statement
# pre-renders the statement into a string, with no driver-supervised parameterization
# useful for testing variant DDL where some fields are not parameterizable
# NOTE: the raw form does its best to quote non-literals where needed, but you may
# have to inject single or double quotes in special cases.
example-raw-cql-stmt:
raw: |
create table if not exist {ksname}.{tblname} ...
# gremlin statement using the fluent API, as it would be written in a client application
example-fluent-graph-stmt:
fluent: >-
g.V().hasLabel("device").has("deviceid", UUID.fromString({deviceid}))
# if imports are not specified, the following is auto imported.
# if imports are specified, you must also provide the __ class if needed
imports:
- org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__
# gremlin statement using the fluent API, as it would be written in a client application
example-fluent-graph-stmt:
fluent: >-
g.V().hasLabel("device").has("deviceid", UUID.fromString({deviceid}))
# if imports are not specified, the following is auto imported.
# if imports are specified, you must also provide the __ class if needed
imports:
- org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__
# gremlin statement using string API (not recommended)
example-raw-gremlin-stmt:
gremlin: >-
g.V().hasLabel("device").has("deviceid", UUID.fromString('{deviceid})')
# gremlin statement using string API (not recommended)
example-raw-gremlin-stmt:
gremlin: >-
g.V().hasLabel("device").has("deviceid", UUID.fromString('{deviceid})')
```
## CQL Op Template - Optional Fields
@@ -183,7 +185,7 @@ params:
# Set the timeout for the operation, from the driver's perspective,
# in seconds. "2 seconds" is the default, but DDL statements, truncate or drop
# statements will generally need more. If you want millisconds, just use
# statements will generally need more. If you want milliseconds, just use
# fractional seconds, like 0.500
timeout: 2.0

4
adapter-cqld4/src/main/resources/cqlgen.md
View File

@@ -1,4 +1,6 @@
# cqlgen - A default CQL workload generator
# cqlgen
cqlgen is a default CQL workload generator
With NB5, the cqld4 driver comes with a workload generator that can be used to generate a
workload yaml from a CQL schema file.

27
adapter-diag/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -37,35 +37,16 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-compiler-plugin</artifactId>-->
<!-- <configuration>-->
<!-- <annotationProcessorPaths>-->
<!-- <path>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>nb-annotations</artifactId>-->
<!-- <version>${version}</version>-->
<!-- </path>-->
<!-- </annotationProcessorPaths>-->
<!-- </configuration>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
</project>

59
adapter-dynamodb/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -34,20 +34,67 @@
<dependencies>
<!-- core dependencies -->
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
<version>1.12.376</version>
<version>1.12.382</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

2
adapter-dynamodb/src/main/resources/dynamodb.md
View File

@@ -1,4 +1,4 @@
# dynamodb driver
# dynamodb
The DynamoDB driver supports a basic set of commands as specified at
[Amazon DynamoDB Docs](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Operations_Amazon_DynamoDB.html).

53
adapter-http/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -33,12 +33,59 @@
This provides the ability to generate HTTP requests with nosqlbench
</description>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<scope>compile</scope>
</dependency>

6
adapter-http/src/main/resources/http.md
View File

@@ -1,4 +1,4 @@
# HTTP driver
# http
This driver allows you to make http requests using the native HTTP client
that is bundled with the JVM. It supports free-form construction of
@@ -23,8 +23,8 @@ Or, you can have a list:
```yaml
# A list of statements
ops:
- http://google.com/
- http://amazon.com/
- http://google.com/
- http://amazon.com/
```
Or you can template the values used in the URI, and even add ratios:

54
adapter-kafka/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -36,18 +36,64 @@
<properties>
<kafka.version>3.3.1</kafka.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>

4
adapter-kafka/src/main/java/io/nosqlbench/adapter/kafka/util/KafkaAdapterMetrics.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -38,7 +38,7 @@ public class KafkaAdapterMetrics implements NBNamedElement {
@Override
public String getName() {
return "S4JAdapterMetrics";
return "KafkaAdapterMetrics";
}
public void initS4JAdapterInstrumentation() {

54
adapter-mongodb/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -36,14 +36,60 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
<version>4.7.1</version>
<version>4.8.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

6
adapter-mongodb/src/main/resources/mongodb.md
View File

@@ -1,4 +1,4 @@
# MongoDB Driver
# MongoDB
This is a driver for MongoDB. It supports the `db.runCommand` API described in [here](https://docs.mongodb.com/manual/reference/command/).
@@ -10,7 +10,9 @@ for details on command structure.
Run a mongodb activity with definitions from activities/mongodb-basic.yaml
... driver=mongodb yaml=activities/mongo-basic.yaml
```shell
nb5 driver=mongodb yaml=activities/mongo-basic.yaml
```
### MongoDB ActivityType Parameters

55
adapter-pulsar/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -41,13 +41,13 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -91,4 +91,51 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

13
adapter-pulsar/src/main/resources/pulsar.md
View File

@@ -1,15 +1,4 @@
# Table of contents
- [1. Overview](#1-overview)
- [1.1. Issues Tracker](#1-1-issues-tracker)
- [2. Execute the NB Pulsar Driver Workload](#2-execute-the-nb-pulsar-driver-workload)
- [2.1. NB Pulsar Driver Yaml File High Level Structure](#2-1-nb-pulsar-driver-yaml-file-high-level-structure)
- [2.2. NB Pulsar Driver Configuration Parameters](#2-2-nb-pulsar-driver-configuration-parameters)
- [2.2.1. Global Level Parameters](#2-2-1-global-level-parameters)
- [2.2.2. Document Level Parameters](#2-2-2-document-level-parameters)
- [3. NB Pulsar Driver OpTemplates](#3-nb-pulsar-driver-optemplates)
- [4. Message Generation and Schema Support](#4-message-generation-and-schema-support)
- [4.1. Message Generation](#4-1-message-generation)
- [4.2. Schema Support](#4-2-schema-support)
# pulsar
# 1. Overview

86
adapter-s4j/pom.xml
View File

@@ -1,3 +1,19 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -7,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -24,28 +40,18 @@
<s4j.version>3.2.0</s4j.version>
</properties>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.7.1</version>
</extension>
</extensions>
</build>
<dependencies>
<!-- core dependencies -->
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.datastax.oss/pulsar-jms -->
@@ -86,4 +92,58 @@
</dependencies>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.7.1</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

20
adapter-s4j/src/main/resources/s4j.md
View File

@@ -1,18 +1,7 @@
---
weight: 0
title: S4J Adapter
title: S4J
---
# S4J Adapter
- [1. Overview](#1-overview)
- [2. Execute NB S4J Workload](#2-execute-nb-s4j-workload)
- [3. NB S4J Driver Configuration Parameter File](#3-nb-s4j-driver-configuration-parameter-file)
- [4. NB S4J Scenario Definition File](#4-nb-s4j-scenario-definition-file)
- [4.1. Document Level Parameters](#4-1-document-level-parameters)
- [4.2. NB S4J Workload Types](#4-2-nb-s4j-workload-types)
- [4.2.1. Publish Messages to a JMS Destination, Queue or Topic](#4-2-1-publish-messages-to-a-jms-destination-queue-or-topic)
- [4.2.2. Receiving Messages from a JMS Destination, Queue or Topic](#4-2-2-receiving-messages-from-a-jms-destination-queue-or-topic)
# 1. Overview
This driver is similar to [NB Pulsar driver](../../../../driver-pulsar/src/main/resources/pulsar.md) that allows NB based workload generation and performance testing against a Pulsar cluster. It also follows a similar pattern to configure and connect to the Pulsar cluster for workload execution.
@@ -23,7 +12,7 @@ However, the major difference is instead of simulating native Pulsar client work
The following is an example of executing a NB S4J workload (defined as *pulsar_s4j.yaml*)
```
```shell
$ <nb_cmd> run driver=s4j cycles=10000 threads=4 num_conn=2 num_session=2 session_mode="client_ack" strict_msg_error_handling="false" web_url=http://localhost:8080 service_url=pulsar://localhost:6650 config=/path/to/nb_s4j_config.properties yaml=/path/to/pulsar_s4j.yaml -vv --logs-dir=s4j_log
```
@@ -153,7 +142,7 @@ The NB S4J statement block for publishing messages to a JMS destination (either
* The default message type (**msg_type**) is "byte". But optionally, you can specify other message types such as "text", "map", etc.
* The message payload (**msg_body**) is the only mandatory field.
```
```yaml
blocks:
msg-produce-block:
ops:
@@ -196,7 +185,8 @@ The generic NB S4J statement block for receiving messages to a JMS destination (
* negative ack/ack timeout/deadletter topic related settings
* The settings here (as the scenario specific settings) will be merged with the
* global settings in *s4j_config.properties* file
```
```yaml
blocks:
msg-produce-block:
ops:

55
adapter-stdout/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -38,18 +38,65 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

36
adapter-stdout/src/main/resources/stdout.md
View File

@@ -1,4 +1,4 @@
# stdout activity type
# stdout
This is an activity type which allows for the generation of data
into to stdout or a file. It reads the standard nosqlbench YAML
@@ -9,15 +9,21 @@ that uses the curly brace token form in statements.
Run a stdout activity named 'stdout-test', with definitions from activities/stdout-test.yaml
... driver=stdout workload=stdout-test
```shell
nb5 driver=stdout workload=stdout-test
```
Only run statement groups which match a tag regex
... driver=stdout workload=stdout-test tags=group:'ddl.*'
```shell
nb5 driver=stdout workload=stdout-test tags=group:'ddl.*'
```
Run the matching 'dml' statements, with 100 cycles, from [1000..1100)
... driver=stdout workload=stdout-test tags=group:'dml.*' cycles=1000..11000 filename=test.csv
```shell
nb5 driver=stdout workload=stdout-test tags=group:'dml.*' cycles=1000..11000 filename=test.csv
```
This last example shows that the cycle range is [inclusive..exclusive),
to allow for stacking test intervals. This is standard across all
@@ -60,15 +66,21 @@ For more details on this format, please refer to the
The statement format for this activity type is a simple string. Tokens between
curly braces are used to refer to binding names, as in the following example:
statements:
- "It is {minutes} past {hour}."
```yaml
ops:
op1: "It is {minutes} past {hour}."
```
If you want to suppress the trailing newline that is automatically added, then
you must either pass `newline=false` as an activity param, or specify it
in the statement params in your config as in:
params:
newline: false
```yaml
ops:
op1:
stmt: "It is {minutes} past {hour}."
newline: false
```
### Auto-generated statements
@@ -76,13 +88,17 @@ If no statement is provided, then the defined binding names are used as-is
to create a CSV-style line format. The values are concatenated with
comma delimiters, so a set of bindings like this:
```yaml
bindings:
one: Identity()
two: NumberNameToString()
```
would create an automatic string template like this:
statements:
- "{one},{two}\n"
```yaml
ops:
op1: "{one},{two}\n"
```
The auto-generation behavior is forced when the format parameter is supplied.

56
adapter-tcp/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -40,14 +40,62 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-stdout</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

2
adapter-tcp/src/main/resources/tcpclient.md
View File

@@ -1,4 +1,4 @@
# Driver: tcpclient
# tcpclient
**tcpclient acts like a _client push_ version of stdout over TCP**

2
adapter-tcp/src/main/resources/tcpserver.md
View File

@@ -1,4 +1,4 @@
# Driver: tcpserver
# tcpserver
**tcpserver acts like a _server push_ version of stdout over TCP**

54
adapters-api/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -39,25 +39,69 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-spectest</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-userlibs</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

12
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawStmtsLoader.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,13 +16,14 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.Logger;
import org.yaml.snakeyaml.Yaml;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import java.util.*;
import java.util.function.Function;
@@ -79,8 +80,9 @@ public class RawStmtsLoader {
}
private RawStmtsDocList parseYaml(Logger logger, String data) {
Yaml yaml = new Yaml();
Iterable<Object> objects = yaml.loadAll(data);
LoadSettings loadSettings = LoadSettings.builder().build();
Load yaml = new Load(loadSettings);
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
for (Object object : objects) {

11
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,13 +16,14 @@
package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import org.apache.logging.log4j.Logger;
import org.yaml.snakeyaml.Yaml;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import java.util.*;
import java.util.function.Function;
@@ -73,8 +74,8 @@ public class RawYamlLoader {
}
private List<Map<String,Object>> parseYaml(Logger logger, String data) {
Yaml yaml = new Yaml();
Iterable<Object> objects = yaml.loadAll(data);
Load yaml = new Load(LoadSettings.builder().build());
Iterable<Object> objects = yaml.loadAllFromString(data);
List<RawStmtsDoc> newDocList = new ArrayList<>();
List<Map<String,Object>> maps = new ArrayList<>();

11
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BundledDriverAdapterDocs.java → adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/docs/BundledDriverAdapterDocs.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,20 +14,17 @@
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityimpl.uniform;
package io.nosqlbench.engine.api.activityimpl.docs;
import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.api.docsapi.DocsNameSpace;
import io.nosqlbench.api.spi.SimpleServiceLoader;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.api.spi.SimpleServiceLoader;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service(value = BundledMarkdownManifest.class, selector = "drivers")
public class BundledDriverAdapterDocs implements BundledMarkdownManifest {

39
adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.specifications;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.nb.annotations.Service;
import java.util.List;
@Service(value = BundledMarkdownManifest.class, selector = "specifications")
public class SpecDocsManifest implements BundledMarkdownManifest {
@Override
public DocsBinder getDocs() {
Docs docs = new Docs().namespace("workload_definition");
List<Content<?>> specfiles = NBIO.classpath().prefix("workload_definition/").extension(".md").list();
for (Content<?> specfile : specfiles) {
docs.addPath(specfile.asPath());
}
return docs;
}
}

6
adapters-api/src/main/java/io/nosqlbench/engine/api/util/AdaptersApiVersionInfo.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -64,6 +64,10 @@ public class AdaptersApiVersionInfo {
String currentField = current_components[i];
if (minField.matches("\\d+")) {
if (currentField.matches("\\d+")) {
if ((Integer.parseInt(currentField) > Integer.parseInt(minField))) {
// We're in a completely newer version
break;
}
if (Integer.parseInt(currentField)<Integer.parseInt(minField)) {
throw new OpConfigError("This workload can only be loaded by a NoSQLBench runtime version " + min_version + " or higher." +
" You are running version " + getVersion());

20
docsys/src/test/java/io/nosqlbench/docsys/core/DocsysMarkdownLoaderEndpointTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/SpecificationDocsManifest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,17 +14,17 @@
* limitations under the License.
*/
package io.nosqlbench.docsys.core;
package io.nosqlbench.engine.api.activityconfig.specifications;
import io.nosqlbench.docsys.endpoints.DocsysMarkdownEndpoint;
import org.junit.jupiter.api.Test;
import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
public class DocsysMarkdownLoaderEndpointTest {
public class SpecificationDocsManifest implements BundledMarkdownManifest {
@Override
public DocsBinder getDocs() {
Docs docs = new Docs();
@Test
public void testDocLoader() {
DocsysMarkdownEndpoint ep = new DocsysMarkdownEndpoint();
String markdownList = ep.getMarkdownList(true);
return docs;
}
}

4
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/UniformWorkloadSpecificationTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/UniformWorkloadSpecificationTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
package io.nosqlbench.engine.api.activityconfig.specifications;
import com.vladsch.flexmark.ast.FencedCodeBlock;
import io.nosqlbench.nb.spectest.core.SpecTest;

5
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/YamlSpecValidator.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,11 +14,12 @@
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig.rawyaml;
package io.nosqlbench.engine.api.activityconfig.specifications;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawYamlLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.nb.spectest.api.STAssemblyValidator;

34
devdocs/updatenotes.md
View File

@@ -9,6 +9,12 @@
* Deprecated old drivers (conflicts, etc)
- version conflicts (removing cql drivers 1.9.* 3.*)
# Performance
* dryrun capability
* slightly improved rate limiter performance
* improved concurrency tracking for activities
* and a bug fix for threads=1 and uncaught errors
## New Topic
* Advanced linearization features
* variable capture in results
@@ -238,3 +244,31 @@ review:
- 394580d66c107127cc68f88cdc64a59e9c481d43
- 0330c18a7ba0904b3b3420b94416b04ee73dd7fb
- 11dd8f62daf3d1603b75cdd85fbb872dbfaac111
* For developers
* New APIs
* DriverAdapter
* SpaceCache
* ParsedOp
* OpMapper
* OpDispenser
* Removal of Phases loop
* Some support for console colors
* Logging subsystem improvements
* Discovery Options
* Project Improvements
* Testing and Documentation Methods
* Literate test examples as documentation
* Integrated test logs are artifacts
* Auto-generated reference docs are not auto-imported
* Major releases are gated by static analysis audits
* We're adding code coverage
* We've added more developers to the project
* Testing Improvements - stability and low-core build systems (gh actions)
* config
* options can be mutually exclusive, like the thread-local rate and the per-activity rate

64
docsys/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -37,12 +37,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<!-- jetty -->
@@ -111,12 +106,12 @@
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>2.3.0.1</version>
<version>4.0.1</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>2.4.0-b180830.0438</version>
<version>4.0.1</version>
</dependency>
<dependency>
@@ -133,9 +128,56 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

5
docsys/src/main/java/io/nosqlbench/docsys/api/DocsysDynamicManifest.java
View File

@@ -17,12 +17,7 @@
package io.nosqlbench.docsys.api;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.docsys.endpoints.DocsysMarkdownEndpoint;
/**
* At runtime, any instances of this service will be used to find
* paths to be shared via the {@link DocsysMarkdownEndpoint}.
*/
public interface DocsysDynamicManifest {
DocsBinder getDocs();
}

25
docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServerApp.java
View File

@@ -16,8 +16,7 @@
package io.nosqlbench.docsys.core;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.docsys.endpoints.DocsysMarkdownEndpoint;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -56,28 +55,6 @@ public class NBWebServerApp implements BundledApp {
logger.info(() -> "generating to directory " + dirpath);
DocsysMarkdownEndpoint dds = new DocsysMarkdownEndpoint();
String markdownList = dds.getMarkdownList(true);
Path markdownCsvPath = dirpath.resolve(Path.of("services/docs/markdown.csv"));
logger.info(() -> "markdown.csv located at " + markdownCsvPath);
Files.createDirectories(markdownCsvPath.getParent());
Files.writeString(markdownCsvPath, markdownList, OVERWRITE);
String[] markdownFileArray = markdownList.split("\n");
for (String markdownFile : markdownFileArray) {
Path relativePath = dirpath.resolve(Path.of("services/docs", markdownFile));
logger.info(() -> "Creating " + relativePath);
Path path = dds.findPath(markdownFile);
// String markdown = dds.getFileByPath(markdownFile);
// Files.writeString(relativePath, markdown, OVERWRITE);
Files.createDirectories(relativePath.getParent());
Files.write(relativePath,Files.readAllBytes(path),OVERWRITE);
}
}
private static void runServer(String[] serverArgs) {

54
docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocServerStatusEndpoint.java
View File

@@ -1,54 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.docsys.endpoints;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.docsys.core.NBWebServer;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.annotations.Maturity;
import jakarta.inject.Singleton;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.Configuration;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@Service(value = WebServiceObject.class, selector = "docserver-status", maturity = Maturity.Deprecated)
@Singleton
@Path("_")
public class DocServerStatusEndpoint implements WebServiceObject {
private final static Logger logger =
LogManager.getLogger(DocServerStatusEndpoint.class);
@Context
private Configuration config;
private String name;
@GET
@Path("status")
@Produces(MediaType.APPLICATION_JSON)
public String getStats() {
NBWebServer s = (NBWebServer) config.getProperty("server");
return s.toString();
}
}

218
docsys/src/main/java/io/nosqlbench/docsys/endpoints/DocsysMarkdownEndpoint.java
View File

@@ -1,218 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.docsys.endpoints;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.api.docsapi.DocsNameSpace;
import io.nosqlbench.docsys.api.WebServiceObject;
import io.nosqlbench.docsys.core.DocsysPathLoader;
import io.nosqlbench.docsys.core.PathWalker;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.annotations.Maturity;
import jakarta.inject.Singleton;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
@Service(value = WebServiceObject.class, selector = "docsys-markdown",maturity = Maturity.Deprecated)
@Singleton
@Path("/services/docs/")
public class DocsysMarkdownEndpoint implements WebServiceObject {
private final static Logger logger = LogManager.getLogger(DocsysMarkdownEndpoint.class);
private DocsBinder docsinfo;
private DocsBinder enabled;
private DocsBinder disabled;
private final AtomicLong version = new AtomicLong(System.nanoTime());
private final Set<String> enables = new HashSet<>();
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("version")
public long getVersion() {
return version.get();
}
/**
* If no enable= parameter is provided, then this call simply provides a map of
* namespaces which are enabled and disabled.
*
* @param enable A set of namespaces to enable, or no provided value to enable all namespaces
* @return A view of the namespaces known to this service
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("namespaces")
public Map<String, Map<String, Set<java.nio.file.Path>>> getNamespaces(
@QueryParam("enable") String enable,
@QueryParam("reload") boolean reload
) {
if (enable!=null && !enable.isEmpty()) {
enables.clear();
enables.addAll(List.of(enable.split("[, ;]")));
}
init(reload);
enable(enables);
return Map.of(
"enabled",enabled.getPathMap(),
"disabled",disabled.getPathMap()
);
}
/**
* @return Provide a list of all files from all enabled namespaces.
*/
@GET
@Produces(MediaType.TEXT_PLAIN)
@Path("allfiles.csv")
public String getAllfilesList(@QueryParam("reload") boolean reload) {
init(reload);
StringBuilder sb = new StringBuilder();
for (java.nio.file.Path path : enabled.getPaths()) {
PathWalker.findAll(path).forEach(f -> {
sb.append(path.relativize(f)).append("\n");
});
}
return sb.toString();
}
/**
* @return Provide a lit of all files from all enabled namespaces
* where the file path ends with '.md'
*/
@GET
@Produces(MediaType.TEXT_PLAIN)
@Path("markdown.csv")
public String getMarkdownList(@QueryParam("reload") boolean reload) {
init(reload);
StringBuilder sb = new StringBuilder();
for (java.nio.file.Path path : enabled.getPaths()) {
PathWalker.findAll(path).forEach(f -> {
if (f.toString().endsWith(".md")) {
sb.append(path.relativize(f)).append("\n");
}
});
}
return sb.toString();
}
/**
* @return Provides a list of all files from all enabled namespaces as a JSON list.
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("list")
public List<String> listFiles(@QueryParam("reload") boolean reload) {
init(reload);
List<String> list = new ArrayList<>();
for (java.nio.file.Path path : enabled.getPaths()) {
PathWalker.findAll(path).forEach(f -> {
java.nio.file.Path relative = path.relativize(f);
list.add(relative.toString());
});
}
return list;
}
/**
* @param pathspec the path as known to the manifest
* @return The contents of a file
*
* @see <A href="https://eclipse-ee4j.github.io/jersey.github.io/documentation/latest/user-guide.html#d0e7648">jersey providers</A>
*
*/
@GET
@Path(value = "{pathspec:.*}")
public Response getFileInPath(@PathParam("pathspec") String pathspec) {
init(false);
try {
java.nio.file.Path path = findPath(pathspec);
String contentType = Files.probeContentType(path);
MediaType mediaType = MediaType.valueOf(contentType);
return Response.ok(Files.newBufferedReader(path), mediaType).build();
} catch (Exception e) {
return Response.serverError().entity(e.getMessage()).build();
}
}
public java.nio.file.Path findPath(String pathspec) {
pathspec = URLDecoder.decode(pathspec, StandardCharsets.UTF_8);
for (java.nio.file.Path path : enabled.getPaths()) {
java.nio.file.Path resolved = path.resolve(pathspec);
if (Files.isDirectory(resolved)) {
throw new RuntimeException("Path is a directory: '" + pathspec + "'");
}
if (Files.exists(resolved)) {
return resolved;
}
}
throw new RuntimeException("Unable to find any valid file at '" + pathspec + "'");
}
private void init(boolean reload) {
if (reload) {
this.enabled = null;
this.disabled = null;
this.docsinfo = null;
}
if (this.docsinfo == null) {
this.docsinfo = DocsysPathLoader.loadDynamicPaths();
version.set(System.nanoTime());
}
if (enabled==null || disabled==null) {
enable(enables);
}
}
private void enable(Set<String> enabled) {
for (DocsNameSpace nsinfo : docsinfo) {
// add namespaces which are neither enabled nor disabled to the default group
if (nsinfo.isEnabledByDefault()) {
if (disabled!=null && disabled.getPathMap().containsKey(nsinfo.getName())) {
continue;
}
enables.add(nsinfo.getName());
}
}
if (enabled.isEmpty()) { // Nothing is enabled or enabled by default, so enable everything
this.enabled = new Docs().merge(docsinfo);
this.disabled = new Docs().asDocsBinder();
} else { // At least one thing was enabled by default, or previously enabled specifically
this.disabled = new Docs().merge(docsinfo);
this.enabled = disabled.remove(enabled);
}
}
}

97
driver-cockroachdb/pom.xml Normal file
View File

@@ -0,0 +1,97 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>4.17.32-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<artifactId>driver-cockroachdb</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>
A CockroachDB ActivityType driver for http://nosqlbench.io/
</description>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>driver-jdbc</artifactId>
<version>4.17.32-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.5.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

94
driver-jdbc/pom.xml Normal file
View File

@@ -0,0 +1,94 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nosqlbench</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.32-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>driver-jdbc</artifactId>
<dependencies>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>3.4.5</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.32-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

53
driver-jmx/pom.xml
View File

@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.32-SNAPSHOT</version>
<version>5.17.0-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -38,11 +38,56 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.32-SNAPSHOT</version>
<version>5.17.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

123
driver-kafka/pom.xml Normal file
View File

@@ -0,0 +1,123 @@
<!--
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.32-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<artifactId>driver-kafka</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>
A Kafka driver for nosqlbench. This provides the ability to inject synthetic data
into a kafka topic.
</description>
<dependencies>
<!-- core dependencies -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.8.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.11.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.confluent/kafka-avro-serializer -->
<dependency>
<groupId>io.confluent</groupId>
<artifactId>kafka-avro-serializer</artifactId>
<version>7.2.1</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.32-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>confluent</id>
<url>https://packages.confluent.io/maven/</url>
</repository>
</repositories>
</project>

141
driver-pulsar/pom.xml Normal file
View File

@@ -0,0 +1,141 @@
<!--
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.32-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<artifactId>driver-pulsar</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>
A Pulsar driver for nosqlbench. This provides the ability to inject synthetic data
into a pulsar system.
</description>
<properties>
<pulsar.version>2.10.2</pulsar.version>
</properties>
<dependencies>
<!-- core dependencies -->
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client</artifactId>
<version>${pulsar.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client-admin</artifactId>
<version>${pulsar.version}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.32-SNAPSHOT</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils -->
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-configuration2 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>2.8.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.11.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

74
engine-api/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -39,31 +39,31 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-spectest</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-userlibs</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -93,11 +93,6 @@
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
@@ -134,4 +129,59 @@
</profile>
</profiles>
<build>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<filtering>false</filtering> <!-- exclusion from defaults -->
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

17
engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/sysperf/SysPerf.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,9 +16,12 @@
package io.nosqlbench.engine.api.activityapi.sysperf;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.yaml.snakeyaml.Yaml;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Dump;
import org.snakeyaml.engine.v2.api.DumpSettings;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import java.io.File;
import java.io.IOException;
@@ -111,8 +114,8 @@ public class SysPerf {
bytes = Files.readAllBytes(cache.toPath());
String perfdata = new String(bytes, CHARSET);
Yaml yaml = new Yaml();
SysPerfData perfinfo = yaml.load(perfdata);
Load yaml = new Load(LoadSettings.builder().build());
SysPerfData perfinfo = (SysPerfData) yaml.loadFromString(perfdata);
cachedData = perfinfo;
logger.info("Loaded previously cached system timing data from " + cache.getCanonicalPath());
return cachedData;
@@ -135,11 +138,11 @@ public class SysPerf {
try {
if (!cache.exists() || forceSave) {
Files.createDirectories(cache.toPath().getParent());
Yaml yaml = new Yaml();
Dump dump = new Dump(DumpSettings.builder().build());
if (cache.exists()) {
cache.delete();
}
String filedata = yaml.dump(cachedData);
String filedata = dump.dumpToString(cachedData);
Files.write(cache.toPath(), filedata.getBytes(CHARSET), StandardOpenOption.CREATE_NEW);
logger.info("Wrote system timing data to cachefile " + cache.getCanonicalPath());
}

67
engine-cli/pom.xml
View File

@@ -1,10 +1,26 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -23,18 +39,63 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-core</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-docker</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

3
engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -72,6 +72,7 @@ public class BasicScriptBuffer implements ScriptBuffer {
case run: // run activity
case await: // await activity
case stop: // stop activity
case forceStop: // force stopping activity
case waitMillis:
sb.append("scenario.").append(cmd).append("\n");

3
engine-cli/src/main/java/io/nosqlbench/engine/cli/Cmd.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -36,6 +36,7 @@ public class Cmd {
run(),
start(),
stop(Arg.of("alias_name")),
forceStop(Arg.of("alias_name")),
script(Arg.of("script_path", s -> s)),
await(Arg.of("alias_name")),
waitMillis(Arg.of("millis_to_wait", Long::parseLong)),

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@@ -25,7 +25,7 @@ import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.logging.NBLogLevel;
import io.nosqlbench.api.metadata.SessionNamer;
import io.nosqlbench.api.metadata.SystemId;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility;
import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
import io.nosqlbench.engine.api.activityapi.input.InputType;

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLICommandParser.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@ public class NBCLICommandParser {
private static final String RUN = "run";
private static final String AWAIT = "await";
private static final String STOP = "stop";
private static final String FORCE_STOP = "forceStop";
private static final String ACTIVITY = "activity";
private static final String SCENARIO = "scenario";
private static final String WAIT_MILLIS = "waitmillis";
@@ -42,7 +43,7 @@ public class NBCLICommandParser {
public static final Set<String> RESERVED_WORDS = new HashSet<>() {{
addAll(
Arrays.asList(
FRAGMENT, SCRIPT, START, RUN, AWAIT, STOP, ACTIVITY, SCENARIO, WAIT_MILLIS
FRAGMENT, SCRIPT, START, RUN, AWAIT, STOP, FORCE_STOP, ACTIVITY, SCENARIO, WAIT_MILLIS
)
);
}};
@@ -63,6 +64,7 @@ public class NBCLICommandParser {
case RUN:
case AWAIT:
case STOP:
case FORCE_STOP:
case WAIT_MILLIS:
cmd = Cmd.parseArg(arglist, canonicalizer);
cmdList.add(cmd);

7
engine-cli/src/main/resources/cli-scripting.md
View File

@@ -88,11 +88,16 @@ To start an activity and then wait for it to complete before continuing:
run <pram>=<value> ...
~~~
To stop an activity by its alias:
To stop an activity by its alias while first waiting for a required thread (motor/slot) entering a specific SlotState:
~~~
stop <activity alias>
~~~
To stop an activity by its alias, without first waiting for a required thread (motor/slot) entering a specific SlotState:
~~~
forceStop <activity alias>
~~~
To wait for a particular activity that has been started to complete before continuing:
~~~
await <activity alias>

63
engine-core/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -38,7 +38,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -81,18 +81,6 @@
<artifactId>profiler</artifactId>
<scope>runtime</scope>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.graalvm.tools</groupId>-->
<!-- <artifactId>chromeinspector</artifactId>-->
<!-- <scope>runtime</scope>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>engine-clients</artifactId>-->
<!-- <version>4.17.32-SNAPSHOT</version>-->
<!-- <scope>compile</scope>-->
<!-- </dependency>-->
<!-- only compile scope -->
@@ -105,6 +93,49 @@
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

30
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -85,9 +85,37 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen
*/
public void stopActivity() {
logger.info(() -> "stopping activity in progress: " + this.getActivityDef().getAlias());
activity.setRunState(RunState.Stopping);
motors.forEach(Motor::requestStop);
tally.awaitNoneOther(RunState.Stopped, RunState.Finished);
shutdownExecutorService(Integer.MAX_VALUE);
tally.awaitNoneOther(RunState.Stopped,RunState.Finished);
activity.setRunState(RunState.Stopped);
logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots");
Annotators.recordAnnotation(Annotation.newBuilder()
.session(sessionId)
.interval(this.startedAt, this.stoppedAt)
.layer(Layer.Activity)
.label("alias", getActivityDef().getAlias())
.label("driver", getActivityDef().getActivityType())
.label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none"))
.detail("params", getActivityDef().toString())
.build()
);
}
/**
* Force stop the motors without trying to wait for the activity to reach stopped/finished state
*/
public void forceStopActivity() {
logger.info(() -> "force stopping activity in progress: " + this.getActivityDef().getAlias());
activity.setRunState(RunState.Stopping);
motors.forEach(Motor::requestStop);
shutdownExecutorService(Integer.MAX_VALUE);
tally.awaitNoneOther(RunState.Stopped,RunState.Finished);

8
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityRuntimeInfo.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -84,9 +84,9 @@ public class ActivityRuntimeInfo implements ProgressCapable {
return this.activity.getRunState();
}
public void stopActivity() {
this.executor.stopActivity();
}
public void stopActivity() { this.executor.stopActivity(); }
public void forceStopActivity() { this.executor.forceStopActivity(); }
public ActivityExecutor getActivityExecutor() {
return executor;

63
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -232,6 +232,67 @@ public class ScenarioController {
}
}
/**
* <p>Force stopping an activity, given an activity def. The only part of the activity def that is important is the
* alias parameter. This method retains the activity def signature to provide convenience for scripting.</p>
* <p>For example, sc.forceStop("alias=foo")</p>
*
* @param activityDef An activity def, including at least the alias parameter.
*/
public synchronized void forceStop(ActivityDef activityDef) {
Annotators.recordAnnotation(Annotation.newBuilder()
.session(this.scenario.getScenarioName())
.now()
.layer(Layer.Activity)
.label("alias", activityDef.getAlias())
.detail("command", "forceStop")
.detail("params", activityDef.toString())
.build());
ActivityRuntimeInfo runtimeInfo = this.activityInfoMap.get(activityDef.getAlias());
if (runtimeInfo == null) {
throw new RuntimeException("could not force stop missing activity:" + activityDef);
}
scenariologger.debug("FORCE STOP " + activityDef.getAlias());
runtimeInfo.forceStopActivity();
}
/**
* <p>Stop an activity, given an activity def map. The only part of the map that is important is the
* alias parameter. This method retains the map signature to provide convenience for scripting.</p>
*
* @param activityDefMap A map, containing at least the alias parameter
*/
public synchronized void forceStop(Map<String, String> activityDefMap) {
ActivityDef ad = new ActivityDef(new ParameterMap(activityDefMap));
forceStop(ad);
}
/**
* Stop an activity, given the name by which it is known already in the scenario. This causes the
* activity to stop all threads, but keeps the thread objects handy for starting again. This can be useful
* for certain testing scenarios in which you want to stop some workloads and start others based on other conditions.
*
* Alternately, you can provide one or more aliases in the same command, and all matching names will be stopped.
*
* @param spec The name of the activity that is already known to the scenario
*/
public synchronized void forceStop(String spec) {
logger.debug("request->STOP '" + spec + "'");
List<String> aliases = Arrays.asList(spec.split("[,; ]"));
List<String> matched = aliases.stream()
.map(String::trim)
.filter(s -> !s.isEmpty())
.flatMap(aspec -> getMatchingAliases(aspec).stream()).collect(Collectors.toList());
for (String alias : matched) {
ActivityDef adef = aliasToDef(alias);
scenariologger.debug("STOP " + adef.getAlias());
forceStop(adef);
}
}
private List<String> getMatchingAliases(String pattern) {
Pattern matcher;

27
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/script/bindings/PolyglotScenarioController.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -148,6 +148,31 @@ public class PolyglotScenarioController {
}
public synchronized void forceStop(Object o) {
if (o instanceof Value) {
forceStop(o);
} else if (o instanceof Map) {
controller.forceStop((Map<String, String>) o);
} else if (o instanceof String) {
controller.forceStop(o.toString());
} else {
throw new RuntimeException("unknown type " + o.getClass().getCanonicalName());
}
}
private synchronized void forceStopValue(Value spec) {
if (spec.isHostObject()) {
controller.forceStop((ActivityDef) spec.asHostObject());
} else if (spec.isString()) {
controller.forceStop(spec.asString());
} else if (spec.hasMembers()) {
controller.forceStop(spec.as(Map.class));
} else {
throw new RuntimeException("unknown base type for graal polyglot: " + spec);
}
}
public synchronized void awaitActivity(Object o) {
this.await(o);
}

68
engine-docker/pom.xml
View File

@@ -1,10 +1,26 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -56,9 +72,57 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

6
engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerHelper.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -65,6 +65,10 @@ public class DockerHelper {
logger.info(() -> "Starting docker with img=" + IMG + ", tag=" + tag + ", name=" + name + ", " +
"ports=" + ports + ", volumes=" + volumeDescList + ", env=" + envList + ", cmds=" + cmdList + ", reload=" + reload);
logger.info("Starting docker with img={}",IMG);
boolean existingContainer = removeExitedContainers(name);
Container containerId = searchContainer(name, reload, tag);

63
engine-docs/pom.xml
View File

@@ -1,10 +1,26 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -28,7 +44,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>docsys</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
@@ -50,6 +66,49 @@
</activation>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>

65
engine-extensions/pom.xml
View File

@@ -1,10 +1,26 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -22,7 +38,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
@@ -40,6 +56,51 @@
</resources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-javadoc-plugin</artifactId>-->

72
engine-rest/pom.xml
View File

@@ -1,10 +1,26 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.32-SNAPSHOT</version>
<version>5.17.0-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -23,21 +39,69 @@
<dependency>
<groupId>io.swagger.parser.v3</groupId>
<artifactId>swagger-parser</artifactId>
<version>2.1.9</version>
<version>2.1.10</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-models</artifactId>
<version>2.2.7</version>
<version>2.2.8</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-cli</artifactId>
<version>4.17.32-SNAPSHOT</version>
<version>5.17.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

16
engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,8 +28,8 @@ import io.nosqlbench.engine.rest.services.WorkSpace;
import io.nosqlbench.engine.rest.services.WorkspaceFinder;
import io.nosqlbench.engine.rest.transfertypes.LiveScenarioView;
import io.nosqlbench.engine.rest.transfertypes.RunScenarioRequest;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import jakarta.inject.Singleton;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
@@ -42,6 +42,8 @@ import java.io.CharArrayWriter;
import java.io.PrintWriter;
import java.util.*;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Service(value = WebServiceObject.class, selector = "scenario-executor")
@Singleton
@@ -155,7 +157,15 @@ public class ScenarioExecutorEndpoint implements WebServiceObject {
LinkedList<String> newargs = new LinkedList<>();
for (String arg : args) {
for (String s : rq.getFilemap().keySet()) {
arg = arg.replaceAll(s, rq.getFilemap().get(s));
Pattern basename = Pattern.compile(s);
String fullyQualifiedName = rq.getFilemap().get(s);
Matcher basenameMatcher = basename.matcher(arg);
StringBuilder newarg = new StringBuilder();
while (basenameMatcher.find()) {
basenameMatcher.appendReplacement(newarg,fullyQualifiedName);
}
basenameMatcher.appendTail(newarg);
arg = newarg.toString();
}
newargs.add(arg);
}

249
mvn-defaults/pom.xml
View File

@@ -19,10 +19,12 @@
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<revision>5.17.0-SNAPSHOT</revision>
<!-- Set this level to override the logging level for tests during build -->
<project.testlevel>INFO</project.testlevel>
<!-- Set this level to override the logging level for tests logging configuration during build -->
@@ -30,46 +32,11 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<javadoc.name>nosqlbench</javadoc.name>
<!-- properties for package versions -->
<commons.compress.version>1.21</commons.compress.version>
<commons.codec.version>1.15</commons.codec.version>
<commons.csv.version>1.9.0</commons.csv.version>
<commons.lang.version>3.12.0</commons.lang.version>
<commons.math3.version>3.6.1</commons.math3.version>
<commons.text.version>1.10.0</commons.text.version>
<greenrobot.version>3.1.0</greenrobot.version>
<docker.java.version>3.2.14</docker.java.version>
<javapoet.version>1.13.0</javapoet.version>
<jaxb.api.version>2.4.0-b180830.0359</jaxb.api.version>
<jaxb.core.version>2.3.0.1</jaxb.core.version>
<jaxb.impl.version>2.4.0-b180830.0438</jaxb.impl.version>
<jacoco.version>0.8.8</jacoco.version>
<lz4.version>1.4.1</lz4.version>
<!-- <metrics.version>4.0.7</metrics.version>-->
<mvel2.version>2.4.14.Final</mvel2.version>
<number.to.words.version>1.0.0</number.to.words.version>
<!-- <slf4j.version>1.7.29</slf4j.version>-->
<snakeyaml.version>1.33</snakeyaml.version>
<snappy.version>1.1.2.6</snappy.version>
<!-- plugin versions -->
<clean.plugin.version>3.1.0</clean.plugin.version>
<compiler.plugin.version>3.8.1</compiler.plugin.version>
<deploy.plugin.version>3.0.0-M1</deploy.plugin.version>
<enforcer.plugin.version>3.0.0-M3</enforcer.plugin.version>
<exec.plugin.version>1.6.0</exec.plugin.version>
<failsafe.plugin.version>3.0.0-M4</failsafe.plugin.version>
<gpg.plugin.version>1.6</gpg.plugin.version>
<install.plugin.version>3.0.0-M1</install.plugin.version>
<jar.plugin.version>3.1.1</jar.plugin.version>
<javadoc.plugin.version>3.1.1</javadoc.plugin.version>
<nexus.staging.plugin.version>1.6.8</nexus.staging.plugin.version>
<release.plugin.version>2.5.3</release.plugin.version>
<resources.plugin.version>3.1.0</resources.plugin.version>
<source.plugin.version>3.0.1</source.plugin.version>
<surefire.plugin.version>3.0.0-M4</surefire.plugin.version>
</properties>
<name>${project.artifactId}</name>
@@ -82,7 +49,7 @@
<licenses>
<license>
<name>The Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
<url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
@@ -100,6 +67,18 @@
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
<version>3.0.14</version>
</dependency>
<dependency>
<groupId>org.snakeyaml</groupId>
<artifactId>snakeyaml-engine</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
@@ -152,7 +131,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>${commons.text.version}</version>
<version>1.10.0</version>
</dependency>
<dependency>
@@ -166,11 +145,6 @@
<version>1.36</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>${snakeyaml.version}</version>
</dependency>
<dependency>
<groupId>com.mitchtalmadge</groupId>
<artifactId>ascii-data</artifactId>
@@ -181,14 +155,14 @@
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>${lz4.version}</version>
<version>1.4.1</version>
</dependency>
<!-- For CQL compression option -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
<version>1.1.2.6</version>
</dependency>
<dependency>
@@ -232,7 +206,7 @@
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-core</artifactId>
<version>${docker.java.version}</version>
<version>3.2.14</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -241,10 +215,16 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.10.0</version>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-transport-okhttp</artifactId>
<version>${docker.java.version}</version>
<version>3.2.14</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -257,7 +237,7 @@
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java</artifactId>
<version>${docker.java.version}</version>
<version>3.2.14</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -276,31 +256,37 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.10</version>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.382</version>
</dependency>
<dependency>
<groupId>com.elega9t</groupId>
<artifactId>number-to-words</artifactId>
<version>${number.to.words.version}</version>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.greenrobot</groupId>
<artifactId>essentials</artifactId>
<version>${greenrobot.version}</version>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons.lang.version}</version>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>com.squareup</groupId>
<artifactId>javapoet</artifactId>
<version>${javapoet.version}</version>
<version>1.13.0</version>
</dependency>
<dependency>
@@ -312,23 +298,23 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>${commons.math3.version}</version>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>${commons.csv.version}</version>
<version>1.9.0</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons.codec.version}</version>
<version>1.15</version>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
<version>${mvel2.version}</version>
<version>2.4.14.Final</version>
</dependency>
<dependency>
@@ -340,7 +326,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>${commons.compress.version}</version>
<version>1.21</version>
</dependency>
<dependency>
@@ -352,36 +338,36 @@
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>${jaxb.core.version}</version>
<version>2.3.0.1</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>${jaxb.impl.version}</version>
<version>2.4.0-b180830.0438</version>
</dependency>
<!-- graalvm -->
<dependency>
<groupId>org.graalvm.sdk</groupId>
<artifactId>graal-sdk</artifactId>
<version>21.3.4</version>
<version>22.2.0</version>
</dependency>
<dependency>
<groupId>org.graalvm.js</groupId>
<artifactId>js</artifactId>
<version>21.3.4</version>
<version>22.2.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.graalvm.js</groupId>
<artifactId>js-scriptengine</artifactId>
<version>21.3.4</version>
<version>22.2.0</version>
</dependency>
<dependency>
<groupId>org.graalvm.tools</groupId>
<artifactId>profiler</artifactId>
<version>21.3.4</version>
<version>22.2.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
@@ -434,48 +420,22 @@
<version>3.4.4</version>
</dependency>
<!-- Removed due to a possible conflict-->
<!-- <dependency>-->
<!-- <groupId>org.apache.logging.log4j</groupId>-->
<!-- <artifactId>slf4j-impl</artifactId>-->
<!-- <version>2.0-alpha2</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>slf4j-ext</artifactId>-->
<!-- <version>1.7.36</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <version>1.7.36</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>slf4j-log4j12</artifactId>-->
<!-- <version>1.7.36</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.23.1</version>
<version>3.24.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<version>1.8.2</version>
<version>1.9.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.8.2</version>
<scope>test</scope>
<version>5.9.2</version>
</dependency>
</dependencies>
@@ -484,10 +444,61 @@
<testResources>
<testResource>
<directory>src/test/resources</directory>
<excludes>
<exclude>log4j2-test.xml</exclude>
</excludes>
<filtering>true</filtering>
</testResource>
<testResource>
<directory>src/test/resources</directory>
<includes>
<include>log4j2-test.xml</include>
</includes>
<filtering>true</filtering>
</testResource>
</testResources>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.7.1</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.2.0</version>
<configuration>
<propertiesEncoding>ISO-8859-1</propertiesEncoding>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.3.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
@@ -498,8 +509,8 @@
<source>17</source>
<release>17</release>
<compilerArgs>
<!-- <compilerArg>-Xdoclint:all</compilerArg>-->
<!-- <compilerArg>-Xlint:all</compilerArg>-->
<!-- <compilerArg>-Xdoclint:all</compilerArg>-->
<!-- <compilerArg>-Xlint:all</compilerArg>-->
</compilerArgs>
</configuration>
</plugin>
@@ -510,7 +521,8 @@
<version>3.0.0-M6</version>
<configuration>
<systemPropertyVariables>
<Log4jContextSelector>org.apache.logging.log4j.core.async.AsyncLoggerContextSelector</Log4jContextSelector>
<Log4jContextSelector>org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
</Log4jContextSelector>
</systemPropertyVariables>
<parallel>methods</parallel>
<excludes>
@@ -553,8 +565,6 @@
</configuration>
</plugin>
<!-- TODO: make sure the IT settings above work for EB IT-->
<!-- Javadoc -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -572,7 +582,7 @@
<additionalOptions>
<additionalOption>-Xdoclint:none</additionalOption>
</additionalOptions>
<!-- <additionalJOption>-Xdoclint:none</additionalJOption>-->
<!-- <additionalJOption>-Xdoclint:none</additionalJOption>-->
<doclint>none</doclint>
</configuration>
<executions>
@@ -665,8 +675,6 @@
</execution>
</executions>
</plugin>
</plugins>
<pluginManagement>
@@ -680,12 +688,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>${release.plugin.version}</version>
<version>3.0.0-M6</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>${gpg.plugin.version}</version>
<version>1.6</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -695,27 +703,27 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.plugin.version}</version>
<version>3.0.0-M4</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>${failsafe.plugin.version}</version>
<version>3.0.0-M4</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${javadoc.plugin.version}</version>
<version>3.1.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${source.plugin.version}</version>
<version>3.0.1</version>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>${nexus.staging.plugin.version}</version>
<version>1.6.8</version>
</plugin>
<plugin>
<groupId>org.antlr</groupId>
@@ -725,45 +733,48 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>${exec.plugin.version}</version>
<version>1.6.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>${enforcer.plugin.version}</version>
<version>3.0.0-M3</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>${clean.plugin.version}</version>
<version>3.1.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>${resources.plugin.version}</version>
<version>3.2.0</version>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>org.jacoco.ant</artifactId>
<version>${jacoco.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${jar.plugin.version}</version>
<version>3.1.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>${install.plugin.version}</version>
<version>3.0.0-M1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>${deploy.plugin.version}</version>
<version>3.0.0-M1</version>
</plugin>
</plugins>
</pluginManagement>
</build>
<developers>
<developer>
<name>Jonathan Shook</name>

66
nb-annotations/pom.xml
View File

@@ -1,3 +1,19 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -5,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -29,4 +45,52 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

68
nb-api/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -17,11 +17,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -58,7 +57,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -72,12 +71,6 @@
<version>0.64.0</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>net.sf.jopt-simple</groupId>
<artifactId>jopt-simple</artifactId>
@@ -97,7 +90,6 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.364</version>
</dependency>
<dependency>
@@ -116,7 +108,6 @@
<artifactId>jmh-generator-annprocess</artifactId>
</dependency>
<!-- TODO: Move this down to virtdata if it is the only module group using it -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
@@ -146,18 +137,65 @@
<artifactId>ascii-data</artifactId>
</dependency>
<dependency>
<groupId>org.graalvm.sdk</groupId>
<artifactId>graal-sdk</artifactId>
</dependency>
<dependency>
<groupId>org.snakeyaml</groupId>
<artifactId>snakeyaml-engine</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

51
nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.apps;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.nb.annotations.Service;
import java.util.Optional;
import java.util.function.ToIntFunction;
public interface BundledApp extends ToIntFunction<String[]> {
int applyAsInt(String[] value);
default String getBundledAppName() {
return this.getClass().getAnnotation(Service.class).selector();
}
default DocsBinder getBundledDocs() {
Docs docs = new Docs().namespace("apps");
String dev_docspath = "app-" + this.getBundledAppName() + "/src/main/resources/docs/" + this.getBundledAppName();
String cp_docspath = "docs/" + this.getBundledAppName();
Optional<Content<?>> bundled_docs = NBIO.local().name(dev_docspath, cp_docspath).first();
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getBundledAppName() + ".md", this.getBundledAppName() + ".md").first();
maindoc.map(Content::asPath).ifPresent(docs::addPath);
return docs.asDocsBinder();
}
}

42
nb-api/src/main/java/io/nosqlbench/api/apps/BundledAppDocs.java Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.apps;
import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.api.spi.SimpleServiceLoader;
import io.nosqlbench.nb.annotations.Maturity;
import io.nosqlbench.nb.annotations.Service;
import java.util.List;
@Service(value = BundledMarkdownManifest.class, selector = "apps")
public class BundledAppDocs implements BundledMarkdownManifest {
@Override
public DocsBinder getDocs() {
DocsBinder docs = new Docs();
SimpleServiceLoader<BundledApp> loader = new SimpleServiceLoader<>(BundledApp.class, Maturity.Any);
List<SimpleServiceLoader.Component<? extends BundledApp>> namedProviders = loader.getNamedProviders();
for (SimpleServiceLoader.Component<? extends BundledApp> namedProvider : namedProviders) {
BundledApp app = namedProvider.provider.get();
DocsBinder bundledDocs = app.getBundledDocs();
docs = docs.merge(bundledDocs);
}
return docs;
}
}

1
nb-api/src/main/java/io/nosqlbench/api/docsapi/BundledMarkdownLoader.java
View File

@@ -18,6 +18,7 @@ package io.nosqlbench.api.docsapi;
import java.util.ServiceLoader;
public class BundledMarkdownLoader {
public static DocsBinder loadBundledMarkdown() {

6
nb-api/src/main/java/io/nosqlbench/api/engine/util/SSLKsFactory.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -45,8 +45,8 @@ public class SSLKsFactory implements NBMapConfigurable {
private static final SSLKsFactory instance = new SSLKsFactory();
private static final Pattern CERT_PATTERN = Pattern.compile("-+BEGIN\\s+.*CERTIFICATE[^-]*-+(?:\\s|\\r|\\n)+([a-z0-9+/=\\r\\n]+)-+END\\s+.*CERTIFICATE[^-]*-+", 2);
private static final Pattern KEY_PATTERN = Pattern.compile("-+BEGIN\\s+.*PRIVATE\\s+KEY[^-]*-+(?:\\s|\\r|\\n)+([a-z0-9+/=\\r\\n]+)-+END\\s+.*PRIVATE\\s+KEY[^-]*-+", 2);
private static final Pattern CERT_PATTERN = Pattern.compile("-+BEGIN\\s+.*CERTIFICATE[^-]*-+(?:\\s|\\r|\\n){1,10}([a-z0-9+/=\\r\\n]+)-+END\\s+.*CERTIFICATE[^-]*-+", Pattern.CASE_INSENSITIVE);
private static final Pattern KEY_PATTERN = Pattern.compile("-+BEGIN\\s+.*PRIVATE\\s+KEY[^-]*-+(?:\\s|\\r|\\n){1,10}([a-z0-9+/=\\r\\n]+)-+END\\s+.*PRIVATE\\s+KEY[^-]*-+", Pattern.CASE_INSENSITIVE);
public static final String SSL = "ssl";
public static final String DEFAULT_TLSVERSION = "TLSv1.2";

15
nb-api/src/main/java/io/nosqlbench/api/markdown/aggregator/MutableFrontMatter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,8 +16,9 @@
package io.nosqlbench.api.markdown.aggregator;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.Yaml;
import org.snakeyaml.engine.v2.api.Dump;
import org.snakeyaml.engine.v2.api.DumpSettings;
import org.snakeyaml.engine.v2.common.FlowStyle;
import java.util.LinkedHashMap;
import java.util.List;
@@ -57,9 +58,9 @@ public class MutableFrontMatter extends LinkedHashMap<String,List<String>> {
}
public String asYaml() {
DumperOptions options = new DumperOptions();
options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
Yaml yaml = new Yaml(options);
return yaml.dump(Map.of(TITLE,getTitle(),WEIGHT,getWeight()));
DumpSettings settings = DumpSettings.builder().setDefaultFlowStyle(FlowStyle.BLOCK).build();
Dump dump = new Dump(settings);
return dump.dumpToString(Map.of(TITLE,getTitle(),WEIGHT,getWeight()));
}
}

2
nb-api/src/main/java/io/nosqlbench/api/markdown/exporter/MarkdownExporter.java
View File

@@ -19,7 +19,7 @@ package io.nosqlbench.api.markdown.exporter;
import io.nosqlbench.api.markdown.aggregator.MarkdownDocs;
import io.nosqlbench.api.markdown.types.DocScope;
import io.nosqlbench.api.markdown.types.MarkdownInfo;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.nb.annotations.Service;
import joptsimple.*;

23
nb-api/src/main/java/io/nosqlbench/api/spi/BundledApp.java
View File

@@ -1,23 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.spi;
import java.util.function.ToIntFunction;
public interface BundledApp extends ToIntFunction<String[]> {
}

53
nb-spectest/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -40,16 +40,17 @@
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-ext-yaml-front-matter</artifactId>
<version>0.62.2</version>
<version>0.64.0</version>
</dependency>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-html2md-converter</artifactId>
<version>0.62.2</version>
<version>0.64.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.10.1</version>
</dependency>
<!-- perf testing -->
@@ -64,6 +65,50 @@
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/resources</directory>

10
nb-spectest/src/main/java/io/nosqlbench/nb/spectest/core/SpecTest.java
View File

@@ -24,6 +24,7 @@ import io.nosqlbench.nb.spectest.api.STBuilderFacets;
import io.nosqlbench.nb.spectest.api.STPathLoader;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@@ -86,6 +87,15 @@ public class SpecTest implements Runnable {
}
}
public List<Path> matchingSpecFiles() {
List<Path> paths = new ArrayList<>();
for (Path path : paths) {
List<Path> matchingPaths = STFileScanner.findMatching(".*\\.md", paths.toArray(new Path[0]));
paths.addAll(matchingPaths);
}
return paths;
}
@Override
public void run() {
Set<STNodeAssembly> testables = new LinkedHashSet<>();

172
nb5/pom.xml
View File

@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -40,13 +40,14 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nbr</artifactId>
<version>4.17.33-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<version>5.17.1-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
</dependency>
<!-- Everything below this line is in common between nb and nb5 -->
@@ -55,85 +56,79 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-tcp</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-mongodb</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-stdout</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-diag</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-dynamodb</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-cqld4</artifactId>
<version>4.17.33-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<version>5.17.1-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-http</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pulsar</artifactId>
<version>4.17.33-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<version>5.17.1-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-s4j</artifactId>
<version>4.17.33-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-tcp</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-kafka</artifactId>
<version>4.17.33-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<version>5.17.1-SNAPSHOT</version>
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <artifactId>slf4j-api</artifactId>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
</dependency>
<dependency>
@@ -201,6 +196,48 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
@@ -214,7 +251,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-mongodb</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
</profile>
@@ -256,6 +293,51 @@
</activation>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<executions>
<execution>
@@ -275,9 +357,5 @@
</plugins>
</build>
</profile>
</profiles>
</project>

67
nbr-examples/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -39,19 +39,70 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nbr</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-diag</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<filtering>false</filtering> <!-- exclusion from defaults -->
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
@@ -66,10 +117,10 @@
</executions>
<configuration>
<parallel>methods</parallel>
<!-- <additionalClasspathElements>-->
<!-- <additionalClasspathElement>${project.basedir}/target/virtdata-userlibs-${project.version}.jar-->
<!-- </additionalClasspathElement>-->
<!-- </additionalClasspathElements>-->
<!-- <additionalClasspathElements>-->
<!-- <additionalClasspathElement>${project.basedir}/target/virtdata-userlibs-${project.version}.jar-->
<!-- </additionalClasspathElement>-->
<!-- </additionalClasspathElements>-->
<includes>
<include>**/*Test*.java</include>
</includes>

60
nbr/pom.xml
View File

@@ -21,7 +21,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -40,31 +40,31 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-cli</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-docs</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-core</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-extensions</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-diag</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
</dependencies>
@@ -73,6 +73,10 @@
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>version.properties</include>
<include>nb_version_info.md</include>
</includes>
<filtering>true</filtering>
</resource>
</resources>
@@ -126,10 +130,50 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>build-nb-appimage</id>

2
nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java
View File

@@ -16,7 +16,7 @@
package io.nosqlbench.api.docsapi.docexporter;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.api.apps.BundledApp;
import io.nosqlbench.nb.annotations.Service;
import joptsimple.OptionParser;
import joptsimple.OptionSet;

34
nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.docsapi.docexporter;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.docsapi.BundledMarkdownManifest;
import io.nosqlbench.api.docsapi.Docs;
import io.nosqlbench.api.docsapi.DocsBinder;
import io.nosqlbench.nb.annotations.Service;
@Service(value=BundledMarkdownManifest.class,selector = "versions")
public class BundledVersionManifest implements BundledMarkdownManifest {
@Override
public DocsBinder getDocs() {
return new Docs()
.namespace("versions")
.addPath(NBIO.classpath().name("nb_version_info.md").one().asPath())
.asDocsBinder();
}
}

10
nbr/src/main/resources/examples/bindings-bytebuffers.yaml
View File

@@ -33,19 +33,17 @@ bindings:
# convert a ByteBuffer to a hex-encoded string with lower case
bb_to_hex_lc: ByteBufferSizedHashed(20); ToHexString(false);
# generate a byte buffer of 1000 bytes, and then compute a MD5
# generate a byte buffer of 1000 bytes, and then compute a SHA-256
# digest into another byte buffer
digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('MD5'); ToHexString();
digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('SHA-256'); ToHexString();
# Md5 digest as above, but using a long as input, short-circuiting
# SHA-256 digest as above, but using a long as input, short-circuiting
# the byte buffer construction spelled out above. This is easier
# to use and faster to generate, although any digest will be
# more intensive to calculate as test data, so only use digests
# where you have specific testing requirements for them.
digest_bb_direct: DigestToByteBuffer('MD5');
digest_bb_direct: DigestToByteBuffer('SHA-256');
# A canned version of the above
long_md5_bb: ToMD5ByteBuffer(); ToHexString();
# The example below show various type-specialized ByteBuffer
# functions which are automatically selected depending on the

9
nbr/src/main/resources/nb_version_info.md Normal file
View File

@@ -0,0 +1,9 @@
# Version Info
The version of NB5 which provided some of the docs content is:
```
version=${project.version}
groupId=${project.groupId}
artifactId=${project.artifactId}
```

28
nbr/src/test/java/io/nosqlbench/cli/testing/ExitStatusIntegrationTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -86,18 +86,18 @@ class ExitStatusIntegrationTests {
assertThat(result.exitStatus).isEqualTo(2);
}
@Test
public void testCloseErrorHandlerOnSpace() {
ProcessInvoker invoker = new ProcessInvoker();
invoker.setLogDir("logs/test");
ProcessResult result = invoker.run("exitstatus_erroronclose", 30,
java, "-jar", JARNAME, "--logs-dir", "logs/test/error_on_close", "run",
"driver=diag", "threads=2", "rate=5", "op=noop", "cycles=10", "erroronclose=true", "-vvv"
);
String stdout = String.join("\n", result.getStdoutData());
String stderr = String.join("\n", result.getStderrData());
assertThat(result.exitStatus).isEqualTo(2);
assertThat(stderr).contains("diag space was configured to throw");
}
// @Test
// public void testCloseErrorHandlerOnSpace() {
// ProcessInvoker invoker = new ProcessInvoker();
// invoker.setLogDir("logs/test");
// ProcessResult result = invoker.run("exitstatus_erroronclose", 30,
// java, "-jar", JARNAME, "--logs-dir", "logs/test/error_on_close", "run",
// "driver=diag", "threads=2", "rate=5", "op=noop", "cycles=10", "erroronclose=true", "-vvv"
// );
// String stdout = String.join("\n", result.getStdoutData());
// String stderr = String.join("\n", result.getStderrData());
// assertThat(result.exitStatus).isEqualTo(2);
// assertThat(stderr).contains("diag space was configured to throw");
// }
}

293
pom.xml
View File

@@ -15,15 +15,15 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<modelVersion>4.0.0</modelVersion>
<artifactId>nosqlbench</artifactId>
<packaging>pom</packaging>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>mvn-defaults</relativePath>
</parent>
@@ -33,8 +33,48 @@
testing systems
</description>
<properties>
<!-- CORE MODULES -->
<module.mvn-defaults>mvn-defaults</module.mvn-defaults>
<module.nb5>nb5</module.nb5>
<module.nbr>nbr</module.nbr>
<module.nbr-examples>nbr-examples</module.nbr-examples>
<module.nb-api>nb-api</module.nb-api>
<module.nb-annotations>nb-annotations</module.nb-annotations>
<module.nb-spectest>nb-spectest</module.nb-spectest>
<module.engine-api>engine-api</module.engine-api>
<module.engine-core>engine-core</module.engine-core>
<module.engine-extensions>engine-extensions</module.engine-extensions>
<module.engine-docker>engine-docker</module.engine-docker>
<module.engine-docs>engine-docs</module.engine-docs>
<module.engine-cli>engine-cli</module.engine-cli>
<module.adapters-api>adapters-api</module.adapters-api>
<!-- driver modules -->
<module.adapter-diag>adapter-diag</module.adapter-diag>
<module.adapter-stdout>adapter-stdout</module.adapter-stdout>
<module.adapter-cqld4>adapter-cqld4</module.adapter-cqld4>
<module.adapter-http>adapter-http</module.adapter-http>
<module.adapter-tcp>adapter-tcp</module.adapter-tcp>
<module.adapter-dynamodb>adapter-dynamodb</module.adapter-dynamodb>
<module.adapter-mongodb>adapter-mongodb</module.adapter-mongodb>
<module.adapter-pulsar>adapter-pulsar</module.adapter-pulsar>
<module.adapter-s4j>adapter-s4j</module.adapter-s4j>
<module.adapter-kafka>adapter-kafka</module.adapter-kafka>
<!-- VIRTDATA MODULES -->
<module.virtdata-api>virtdata-api</module.virtdata-api>
<module.virtdata-lang>virtdata-lang</module.virtdata-lang>
<module.virtdata-realdata>virtdata-realdata</module.virtdata-realdata>
<module.virtdata-lib-basics>virtdata-lib-basics</module.virtdata-lib-basics>
<module.virtdata-lib-random>virtdata-lib-random</module.virtdata-lib-random>
<module.virtdata-lib-curves4>virtdata-lib-curves4</module.virtdata-lib-curves4>
<module.virtdata-lib-realer>virtdata-lib-realer</module.virtdata-lib-realer>
<module.virtdata-userlibs>virtdata-userlibs</module.virtdata-userlibs>
</properties>
<modules>
<!-- TODO - consider using properties in the module defs. -->
<!-- CORE MODULES -->
<module>mvn-defaults</module>
<module>nb5</module>
@@ -49,8 +89,6 @@
<module>engine-docker</module>
<module>engine-docs</module>
<module>engine-cli</module>
<!-- <module>engine-rest</module>-->
<!-- <module>engine-clients</module>-->
<module>adapters-api</module>
<!-- driver modules -->
@@ -67,7 +105,6 @@
<module>adapter-cockroachdb</module>
<!-- VIRTDATA MODULES -->
<module>virtdata-api</module>
<module>virtdata-lang</module>
<module>virtdata-realdata</module>
@@ -114,6 +151,250 @@
<!-- <executions>-->
<!-- </executions>-->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.6</version>
<executions>
<execution>
<phase>post-integration-test</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<echo message="Generating NB Test Coverage Reports" />
<taskdef name="report" classname="org.jacoco.ant.ReportTask">
<classpath path="${basedir}/target/jacoco-jars/org.jacoco.ant.jar" />
</taskdef>
<mkdir dir="${basedir}/target/coverage-report" />
<report>
<executiondata>
<fileset dir="${basedir}/${module.nbr}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.nb-api}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.nb-annotations}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.nb-spectest}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.engine-api}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.engine-core}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.engine-extensions}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.engine-docker}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.engine-docs}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.engine-cli}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapters-api}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-diag}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-stdout}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-cqld4}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-http}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-tcp}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-dynamodb}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-mongodb}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-pulsar}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-s4j}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.adapter-kafka}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-api}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lang}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-realdata}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-basics}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-random}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-curves4}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-realer}/target">
<include name="jacoco.exec" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-userlibs}/target">
<include name="jacoco.exec" />
</fileset>
</executiondata>
<structure name="NB Coverage">
<group name="jacoco-nb-all-modules">
<classfiles>
<fileset dir="${basedir}/${module.nb-api}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.nb-annotations}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.nb-spectest}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.engine-api}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.engine-core}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.engine-extensions}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.engine-docker}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.engine-docs}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.engine-cli}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapters-api}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-diag}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-stdout}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-cqld4}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-http}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-tcp}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-dynamodb}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-mongodb}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-pulsar}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-s4j}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.adapter-kafka}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-api}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lang}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-realdata}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-basics}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-random}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-curves4}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-lib-realer}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
<fileset dir="${basedir}/${module.virtdata-userlibs}/target/classes">
<include name="io/nosqlbench/**/*.class" />
</fileset>
</classfiles>
<sourcefiles encoding="UTF-8">
<fileset dir="${basedir}/${module.nb-api}/src/main/java" />
<fileset dir="${basedir}/${module.nb-annotations}/src/main/java" />
<fileset dir="${basedir}/${module.nb-spectest}/src/main/java" />
<fileset dir="${basedir}/${module.engine-api}/src/main/java" />
<fileset dir="${basedir}/${module.engine-core}/src/main/java" />
<fileset dir="${basedir}/${module.engine-extensions}/src/main/java" />
<fileset dir="${basedir}/${module.engine-docker}/src/main/java" />
<fileset dir="${basedir}/${module.engine-docs}/src/main/java" />
<fileset dir="${basedir}/${module.engine-cli}/src/main/java" />
<fileset dir="${basedir}/${module.adapters-api}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-diag}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-stdout}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-cqld4}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-http}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-tcp}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-dynamodb}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-mongodb}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-pulsar}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-s4j}/src/main/java" />
<fileset dir="${basedir}/${module.adapter-kafka}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-api}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-lang}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-realdata}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-lib-basics}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-lib-random}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-lib-curves4}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-lib-realer}/src/main/java" />
<fileset dir="${basedir}/${module.virtdata-userlibs}/src/main/java" />
</sourcefiles>
</group>
</structure>
<html destdir="${basedir}/target/coverage-report/html" />
<xml destfile="${basedir}/target/coverage-report/nb-coverage-report.xml" />
<csv destfile="${basedir}/target/coverage-report/nb-coverage-report.csv" />
</report>
</target>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.jacoco</groupId>
<artifactId>org.jacoco.ant</artifactId>
<version>${jacoco.version}</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
@@ -123,7 +404,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>2.9</version>
<version>3.4.1</version>
</plugin>
</plugins>
</reporting>

68
virtdata-api/pom.xml
View File

@@ -1,3 +1,19 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -7,7 +23,7 @@
<parent>
<groupId>io.nosqlbench</groupId>
<artifactId>mvn-defaults</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -23,14 +39,14 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<artifactId>nb-api</artifactId>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-lang</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
@@ -45,5 +61,51 @@
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

4
virtdata-api/src/main/java/io/nosqlbench/virtdata/api/annotations/ExampleData.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,7 +34,7 @@ import java.util.regex.Pattern;
public class ExampleData {
public static Pattern CTOR_PATTERN = Pattern.compile("(?<funcname>[^)]+)\\((?<args>.+)\\)");
public static Pattern VALS_PATTERN = Pattern.compile("\\[(?<values>-?\\d+([,-. ]+-?\\d+)*)]");
public static Pattern VALS_PATTERN = Pattern.compile("\\[(?<values>-?\\d+([,\\-. ]{1,2}-?\\d+)*)]");
private final Pattern COMMA_VALS = Pattern.compile("\\[(?<vals>-?\\d+(,-?\\d+)*)]");
private final Pattern RANGE_VALS = Pattern.compile("\\[(?<from>-?\\d+)\\.\\.(?<to>-?\\d+)( +(?<step>-?\\d+))?]");

128
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/CompatibilityFixups.java
View File

@@ -1,128 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.core.bindings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CompatibilityFixups {
private final static Logger logger = LogManager.getLogger(CompatibilityFixups.class);
// Not all of these are simple upper-case changes
private final static Map<String, String> funcs = new HashMap<String, String>() {{
put("log_normal", "LogNormal");
put("normal", "Normal");
put("levy", "Levy");
put("nakagami", "Nakagami");
put("exponential", "Exponential");
put("logistic", "Logistic");
put("laplace", "Laplace");
put("cauchy", "Cauchy");
put("f", "F");
put("t", "T");
put("weibull", "Weibull");
put("chi_squared", "ChiSquared");
put("gumbel", "Gumbel");
put("beta", "Beta");
put("pareto", "Pareto");
put("gamma", "Gamma");
put("uniform_real", "Uniform");
put("uniform_integer", "Uniform");
put("hypergeometric", "Hypergeometric");
put("geometric", "Geometric");
put("poisson", "Poisson");
put("zipf", "Zipf");
put("binomial", "Binomial");
put("pascal", "Pascal");
}};
private static final String MAPTO = "mapto_";
private static final String HASHTO = "hashto_";
private static final String COMPUTE = "compute_";
private static final String INTERPOLATE = "interpolate_";
private final static Pattern oldcurve = Pattern.compile("(?<name>\\b[\\w_]+)(?<lparen>\\()(?<args>.*?)(?<rparen>\\))");
private final static CompatibilityFixups instance = new CompatibilityFixups();
public static String fixup(String spec) {
String fixed = instance.fix(spec);
if (!fixed.equals(spec)) {
logger.warn(spec + "' was preprocessed to '" + fixed + "'. Please change to the new one to avoid this warning.");
}
return fixed;
}
public String fix(String spec) {
if (spec == null) {
throw new RuntimeException("Unable to fixup a spec that is null");
}
// Fixup curve ctors. These are not HOF, so local matching will work fine. However, they could occur multiple
// times within an HOF, so multiple replace is necessary.
Matcher matcher = oldcurve.matcher(spec);
StringBuilder out = new StringBuilder(spec.length());
int start = 0;
while (matcher.find()) {
out.append(spec, start, matcher.start());
String replacement = fixCurveCall(matcher.group("name"), matcher.group("args"));
out.append(replacement);
start = matcher.end();
}
out.append(spec.substring(start));
return out.toString();
}
private String fixCurveCall(String name, String args) {
boolean map = false;
boolean compute = false;
if (name.contains(MAPTO)) {
name = name.replaceAll(MAPTO, "");
map = true;
}
if (name.contains(HASHTO)) {
name = name.replaceAll(HASHTO, "");
map = false;
}
if (name.contains(COMPUTE)) {
name = name.replaceAll(COMPUTE, "");
compute = true;
}
if (name.contains(INTERPOLATE)) {
name = name.replaceAll(INTERPOLATE, "");
compute = false;
}
String nameReplacement = funcs.get(name);
if (nameReplacement != null) {
name = nameReplacement;
args = map ? args + ",'map'" : args + ",'hash'";
args = compute ? args + ",'compute'" : args + ",'interpolate'";
}
return name + "(" + args + ")";
}
}

5
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtData.java
View File

@@ -97,7 +97,6 @@ public class VirtData {
* @return An optional function which will be empty if the function could not be resolved.
*/
public static <T> Optional<DataMapper<T>> getOptionalMapper(String flowSpec, Map<String,?> config) {
flowSpec = CompatibilityFixups.fixup(flowSpec);
VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(flowSpec);
if (parseResult.throwable != null) {
throw new RuntimeException("Error while parsing binding specification '" + flowSpec +"': "+ parseResult.throwable);
@@ -118,7 +117,6 @@ public class VirtData {
public static ResolverDiagnostics getMapperDiagnostics(String flowSpec, Map<String,Object> config) {
try {
flowSpec = CompatibilityFixups.fixup(flowSpec);
VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(flowSpec);
if (parseResult.throwable != null) {
throw new RuntimeException(parseResult.throwable);
@@ -156,7 +154,7 @@ public class VirtData {
final String originalflowSpec,
Class<?> clazz,
Map<String,Object> config) {
String flowSpec = CompatibilityFixups.fixup(originalflowSpec);
String flowSpec = originalflowSpec;
VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(flowSpec);
if (parseResult.throwable != null) {
throw new RuntimeException(parseResult.throwable);
@@ -207,7 +205,6 @@ public class VirtData {
}
public static <T> Optional<T> getOptionalFunction(String flowSpec, Class<? extends T> functionType, Map<String,Object> config) {
flowSpec = CompatibilityFixups.fixup(flowSpec);
Class<?> requiredInputType = FunctionTyper.getInputClass(functionType);
Class<?> requiredOutputType = FunctionTyper.getResultClass(functionType);

51
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/CompatibilityFixupsTest.java
View File

@@ -1,51 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.core;
import io.nosqlbench.virtdata.core.bindings.CompatibilityFixups;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class CompatibilityFixupsTest {
@Test
public void testInlineChange() {
assertThat(CompatibilityFixups.fixup("Hash(); uniform_integer(0,1000000000); ToString() -> String"))
.isEqualTo("Hash(); Uniform(0,1000000000,'hash','interpolate'); ToString() -> String");
}
@Test
public void testFixupModifiers() {
assertThat(CompatibilityFixups.fixup("compute_levy(ASDF)")).isEqualTo("Levy(ASDF,'hash','compute')");
assertThat(CompatibilityFixups.fixup("interpolate_levy(ASDF)")).isEqualTo("Levy(ASDF,'hash','interpolate')");
assertThat(CompatibilityFixups.fixup("mapto_levy(ASDF)")).isEqualTo("Levy(ASDF,'map','interpolate')");
assertThat(CompatibilityFixups.fixup("hashto_levy(ASDF)")).isEqualTo("Levy(ASDF,'hash','interpolate')");
}
@Test
public void testFixupNames() {
assertThat(CompatibilityFixups.fixup("gamma(foo)")).isEqualTo("Gamma(foo,'hash','interpolate')");
assertThat(CompatibilityFixups.fixup("mapto_uniform_integer(foo)")).isEqualTo("Uniform(foo,'map','interpolate')");
assertThat(CompatibilityFixups.fixup("hashto_uniform_real(foo)")).isEqualTo("Uniform(foo,'hash','interpolate')");
}
@Test
public void testParsingSanity() {
assertThat(CompatibilityFixups.fixup("long -> Add(5) -> long")).isEqualTo("long -> Add(5) -> long");
}
}

47
virtdata-lang/pom.xml
View File

@@ -1,5 +1,5 @@
<!--
~ Copyright (c) 2022 nosqlbench
~ Copyright (c) 2022-2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -62,7 +62,6 @@
</execution>
</executions>
</plugin>
<plugin>
<version>3.2.0</version>
<groupId>org.apache.maven.plugins</groupId>
@@ -82,6 +81,48 @@
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

62
virtdata-lib-basics/pom.xml
View File

@@ -1,3 +1,19 @@
<!--
~ Copyright (c) 2023 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -7,7 +23,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@@ -20,7 +36,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>virtdata-api</artifactId>
<version>4.17.33-SNAPSHOT</version>
<version>5.17.1-SNAPSHOT</version>
</dependency>
<dependency>
@@ -61,6 +77,48 @@
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
<execution>
<id>jacoco-check</id>
<phase>verify</phase>
<goals>
<goal>check</goal>
</goals>
<configuration>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.00</minimum>
<maximum>1.00</maximum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

61
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,9 +20,12 @@ import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.commons.codec.digest.MessageDigestAlgorithms;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.function.LongFunction;
import java.util.function.Supplier;
@@ -33,44 +36,58 @@ import java.util.stream.Collectors;
@ThreadSafeMapper
public class DigestToByteBuffer implements LongFunction<ByteBuffer> {
private transient static ThreadLocal<TL_State> tl_state;
private static final Logger logger = LogManager.getLogger(DigestToByteBuffer.class);
private static ThreadLocal<ThreadLocalState> state = null;
public DigestToByteBuffer(String digestType) {
for (String digestName : MessageDigestAlgorithms.values()) {
if (digestName.equals(digestType)) {
Supplier<MessageDigest> mds = () -> getDigest(digestName);
tl_state = ThreadLocal.withInitial(() -> new TL_State(mds));
break;
}
if (!Arrays.asList(MessageDigestAlgorithms.values()).contains(digestType)) {
throw new RuntimeException("A digest of type " + digestType +
" was not found. Select a digest type from: "
+ Arrays.stream(MessageDigestAlgorithms.values()).
collect(Collectors.joining(",", "[", "]")));
}
if (tl_state==null) {
tl_state = ThreadLocal.withInitial(() -> new TL_State(() -> getDigest(digestType)));
}
}
private static MessageDigest getDigest(String type) {
try {
return MessageDigest.getInstance(type);
if (digestType.equalsIgnoreCase("md5") ||digestType.equalsIgnoreCase("md2") ) {
logger.warn("Not recommended to use 'MD5 or MD2'. A stronger message digest algorithm is recommended.");
}
if (state != null) {
state.remove();
}
final Supplier<MessageDigest> mds = () -> {
try {
return MessageDigest.getInstance(digestType);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
};
state = ThreadLocal.withInitial(() -> new ThreadLocalState(mds));
} catch (Exception e) {
throw new RuntimeException("A digest of type " + type + " was not found. Select a digest type from: " +
Arrays.stream(MessageDigestAlgorithms.values()).collect(Collectors.joining(",", "[", "]")));
throw new RuntimeException("Unexpected error: ", e);
}
}
@Override
public ByteBuffer apply(long value) {
TL_State state = tl_state.get();
state.buf.putLong(0,value);
byte[] digest = state.digest.digest(state.buf.array());
return ByteBuffer.wrap(digest);
if (DigestToByteBuffer.state != null) {
final ThreadLocalState tlState = DigestToByteBuffer.state.get();
tlState.buf.putLong(0, value);
byte[] digest = tlState.digest.digest(tlState.buf.array());
return ByteBuffer.wrap(digest);
}
throw new RuntimeException("Unable to apply long value as state is not initialized.");
}
private final static class TL_State {
private static final class ThreadLocalState {
private final MessageDigest digest;
private final ByteBuffer buf = ByteBuffer.allocate(Long.BYTES);
public TL_State(Supplier<MessageDigest> mds) {
public ThreadLocalState(Supplier<MessageDigest> mds) {
digest = mds.get();
}
}

49
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,48 +22,41 @@ import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.function.LongFunction;
/**
* Converts the byte image of the input long to a MD5 digest in ByteBuffer form.
* Deprecated usage due to unsafe MD5 digest.
* Replaced with DigestToByteBuffer with MD5 when absolutely needed for existing NB tests.
* However, stronger encryption algorithms (e.g. SHA-256) are recommended due to MD5's limitations.
*/
@Categories({Category.conversion,Category.premade})
@Categories({Category.conversion, Category.premade})
@ThreadSafeMapper
@Deprecated(since = "NB5", forRemoval = true)
public class ToMD5ByteBuffer implements LongFunction<ByteBuffer> {
private final MessageDigest md5;
private transient static final ThreadLocal<TLState> tl_state = ThreadLocal.withInitial(TLState::new);
@Example({"MD5ByteBuffer()","convert the a input to an md5 digest of its bytes"})
/**
* Deprecated usage due to unsafe MD5 digest.
* Use the DigestToByteBuffer with alternatives other than MD5.
*/
@Example({"MD5ByteBuffer()", "convert the a input to an md5 digest of its bytes"})
@Deprecated
public ToMD5ByteBuffer() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
throw new RuntimeException("No longer available. Please use the DigestToByteBuffer with " +
"alternatives other than MD5");
}
/**
* Deprecated usage due to unsafe MD5 digest used in this class.
* Use the DigestToByteBuffer with alternatives other than MD5.
*/
@Override
@Deprecated
public ByteBuffer apply(long value) {
TLState state = tl_state.get();
state.md5.reset();
state.bytes.putLong(0,value);
byte[] digest = md5.digest(state.bytes.array());
return ByteBuffer.wrap(digest);
throw new RuntimeException("No longer available. Please use the DigestToByteBuffer with " +
"alternatives other than MD5");
}
private final static class TLState {
public final ByteBuffer bytes = ByteBuffer.allocate(160);
public final MessageDigest md5;
public TLState() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
}

71
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java
View File

@@ -1,71 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.commons.codec.binary.Hex;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.function.LongFunction;
/**
* Computes the MD5 digest of the byte image of the input long, and
* returns it in hexadecimal String form.
*/
@Categories(Category.conversion)
@ThreadSafeMapper
public class MD5HexString implements LongFunction<String> {
private final MessageDigest md5;
private static final transient ThreadLocal<TLState> tl_state = ThreadLocal.withInitial(TLState::new);
@Example({"MD5String()","Convert a long input to an md5 digest over its bytes, and then to a hexadecimal string."})
public MD5HexString() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
@Override
public String apply(long value) {
TLState state = tl_state.get();
state.bytes.putLong(0,value);
byte[] digest = md5.digest(state.bytes.array());
String hexDigest = Hex.encodeHexString(digest);
return hexDigest;
}
private final static class TLState {
public final ByteBuffer bytes = ByteBuffer.allocate(16);
public final MessageDigest md5;
public TLState() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
}

52
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,13 +23,12 @@ import org.junit.jupiter.api.Test;
import java.nio.ByteBuffer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.*;
public class DigestToByteBufferTest {
class DigestToByteBufferTest {
@Test
public void testWithMD5() {
void testWithMD5() {
DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.MD5);
ByteBuffer digest = d1.apply(233423L);
byte[] bytes;
@@ -43,7 +42,7 @@ public class DigestToByteBufferTest {
}
@Test
public void testWithSHA1() {
void testWithSHA1() {
DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.SHA_1);
ByteBuffer digest = d1.apply(233423L);
byte[] bytes;
@@ -57,10 +56,45 @@ public class DigestToByteBufferTest {
}
@Test
public void testInvalidName() {
DigestToByteBuffer d1 = new DigestToByteBuffer("Whoops");
void testInvalidNames() {
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> d1.apply(233423L));
.isThrownBy(() -> new DigestToByteBuffer("Whoops"));
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> new DigestToByteBuffer(""));
}
@Test
void testInstances() {
DigestToByteBuffer sha256 = new DigestToByteBuffer("SHA-256");
DigestToByteBuffer sha512 = new DigestToByteBuffer("SHA-512");
try {
ByteBuffer sha256Digest = sha256.apply(8675309L);
ByteBuffer sha512Digest = sha512.apply(8675309L);
byte[] bytesFromSha256;
byte[] bytesFromSha512;
try {
bytesFromSha256 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b");
bytesFromSha512 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b");
} catch (DecoderException e) {
throw new RuntimeException(e);
}
// System.out.println(Hex.encodeHexString(sha256Digest));
// System.out.println(Hex.encodeHexString(sha512Digest));
assertThat(sha256Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha256));
assertThat(sha512Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha512));
} catch(Exception e) {
fail("unexpected exception found.");
}
}
}

Some files were not shown because too many files have changed in this diff Show More