Merge pull request #105 from nosqlbench/nosqlbench-104

Nosqlbench 104

activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/to_daterange/DateRangeDuring.java

@@ -0,0 +1,46 @@
+package io.nosqlbench.activitytype.cql.datamappers.functions.to_daterange;
+
+import com.datastax.driver.dse.search.DateRange;
+import io.nosqlbench.virtdata.annotations.Example;
+import io.nosqlbench.virtdata.annotations.ThreadSafeMapper;
+
+import java.util.Date;
+import java.util.function.Function;
+import java.util.function.LongFunction;
+
+/**
+ * Takes an input as a reference point in epoch time, and converts it to a DateRange,
+ * with the bounds set to the lower and upper timestamps which align to the
+ * specified precision. You can use any of these precisions to control the bounds
+ * around the provided timestamp:
+ * <ul>
+ *     <LI>millisecond</LI>
+ *     <LI>second</LI>
+ *     <LI>minute</LI>
+ *     <LI>hour</LI>
+ *     <li>day</li>
+ *     <li>month</li>
+ *     <li>year</li>
+ * </ul>
+ */
+@ThreadSafeMapper
+public class DateRangeDuring implements LongFunction<DateRange> {
+
+    private final com.datastax.driver.dse.search.DateRange.DateRangeBound.Precision precision;
+
+    @Example({"DateRangeDuring('millisecond')}","Convert the incoming millisecond to an equivalent DateRange"})
+    @Example({"DateRangeDuring('minute')}","Convert the incoming millisecond to a DateRange for the minute in which the " +
+        "millisecond falls"})
+    public DateRangeDuring(String precision) {
+        this.precision = com.datastax.driver.dse.search.DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+    }
+
+    @Override
+    public DateRange apply(long value) {
+        Date date = new Date(value);
+        com.datastax.driver.dse.search.DateRange.DateRangeBound lower = com.datastax.driver.dse.search.DateRange.DateRangeBound.lowerBound(date, precision);
+        com.datastax.driver.dse.search.DateRange.DateRangeBound upper = com.datastax.driver.dse.search.DateRange.DateRangeBound.upperBound(date, precision);
+        com.datastax.driver.dse.search.DateRange dateRange = new com.datastax.driver.dse.search.DateRange(lower, upper);
+        return dateRange;
+    }
+}

activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/to_daterange/DateRangeFunc.java

@@ -0,0 +1,60 @@
+package io.nosqlbench.activitytype.cql.datamappers.functions.to_daterange;
+
+import com.datastax.driver.dse.search.DateRange;
+import io.nosqlbench.virtdata.annotations.Example;
+import io.nosqlbench.virtdata.annotations.ThreadSafeMapper;
+
+import java.util.Date;
+import java.util.function.Function;
+import java.util.function.LongFunction;
+import java.util.function.LongUnaryOperator;
+
+/**
+ * Uses the precision and the two functions provided to create a DateRange.
+ * You can use any of these precisions to control the bounds
+ * around the provided timestamp:
+ * <ul>
+ *     <LI>millisecond</LI>
+ *     <LI>second</LI>
+ *     <LI>minute</LI>
+ *     <LI>hour</LI>
+ *     <li>day</li>
+ *     <li>month</li>
+ *     <li>year</li>
+ * </ul>
+ */
+@ThreadSafeMapper
+public class DateRangeFunc implements LongFunction<DateRange> {
+
+    private final DateRange.DateRangeBound.Precision precision;
+    private final LongUnaryOperator lower;
+    private final LongUnaryOperator upper;
+
+    public DateRangeFunc(String precision, LongUnaryOperator lower, LongUnaryOperator upper) {
+        this.precision = DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+        this.lower = lower;
+        this.upper = upper;
+    }
+    public DateRangeFunc(String precision, LongFunction<Long> lower, LongFunction<Long> upper) {
+        this.precision = DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+        this.lower = lower::apply;
+        this.upper = upper::apply;
+    }
+    public DateRangeFunc(String precision, Function<Long,Long> lower, Function<Long,Long> upper) {
+        this.precision = DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+        this.lower = lower::apply;
+        this.upper = upper::apply;
+    }
+
+
+
+    @Override
+    public DateRange apply(long value) {
+        Date lowerDate = new Date(lower.applyAsLong(value));
+        DateRange.DateRangeBound lower = DateRange.DateRangeBound.lowerBound(lowerDate,precision);
+        Date upperDate = new Date(upper.applyAsLong(value));
+        DateRange.DateRangeBound upper = DateRange.DateRangeBound.upperBound(lowerDate,precision);
+        DateRange dateRange = new DateRange(lower, upper);
+        return dateRange;
+    }
+}

activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/to_daterange/DateRangeOnOrAfter.java

@@ -0,0 +1,48 @@
+package io.nosqlbench.activitytype.cql.datamappers.functions.to_daterange;
+
+import com.datastax.driver.dse.search.DateRange;
+import io.nosqlbench.virtdata.annotations.Example;
+import io.nosqlbench.virtdata.annotations.ThreadSafeMapper;
+
+import java.util.Date;
+import java.util.function.Function;
+import java.util.function.LongFunction;
+
+/**
+ * Takes an input as a reference point in epoch time, and converts it to a DateRange,
+ * with the lower bounds set to the lower bound of the precision and millisecond
+ * provided, and with no upper bound.
+ * You can use any of these precisions to control the bounds
+ * around the provided timestamp:
+ * <ul>
+ *     <LI>millisecond</LI>
+ *     <LI>second</LI>
+ *     <LI>minute</LI>
+ *     <LI>hour</LI>
+ *     <li>day</li>
+ *     <li>month</li>
+ *     <li>year</li>
+ * </ul>
+ */
+@ThreadSafeMapper
+public class DateRangeOnOrAfter implements LongFunction<DateRange> {
+
+    private final DateRange.DateRangeBound.Precision precision;
+
+    @Example({"DateRangeOnOrAfter('millisecond')}","Convert the incoming millisecond to an match any time on or after"})
+    @Example({"DateRangeOnOrAfter('minute')}","Convert the incoming millisecond to mach any time on or after the" +
+        " minute in which the " +
+        "millisecond falls"})
+    public DateRangeOnOrAfter(String precision) {
+        this.precision = DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+    }
+
+    @Override
+    public DateRange apply(long value) {
+        Date date = new Date(value);
+        DateRange.DateRangeBound lower = DateRange.DateRangeBound.lowerBound(date, precision);
+        DateRange.DateRangeBound upper = DateRange.DateRangeBound.UNBOUNDED;
+        DateRange dateRange = new DateRange(lower, upper);
+        return dateRange;
+    }
+}

activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/to_daterange/DateRangeOnOrBefore.java

@@ -0,0 +1,47 @@
+package io.nosqlbench.activitytype.cql.datamappers.functions.to_daterange;
+
+import com.datastax.driver.dse.search.DateRange;
+import io.nosqlbench.virtdata.annotations.Example;
+import io.nosqlbench.virtdata.annotations.ThreadSafeMapper;
+
+import java.util.Date;
+import java.util.function.Function;
+import java.util.function.LongFunction;
+
+/**
+ * Takes an input as a reference point in epoch time, and converts it to a DateRange,
+ * with the upper bound set to the upper bound of the precision and millisecond
+ * provided, and with no lower bound.
+ * You can use any of these precisions to control the bounds
+ * around the provided timestamp:
+ * <ul>
+ *     <LI>millisecond</LI>
+ *     <LI>second</LI>
+ *     <LI>minute</LI>
+ *     <LI>hour</LI>
+ *     <li>day</li>
+ *     <li>month</li>
+ *     <li>year</li>
+ * </ul>
+ */
+@ThreadSafeMapper
+public class DateRangeOnOrBefore implements LongFunction<DateRange> {
+
+    private final DateRange.DateRangeBound.Precision precision;
+
+    @Example({"DateRangeOnOrBefore('millisecond')}","Convert the incoming millisecond to match anything on or before it."})
+    @Example({"DateRangeOnOrBefore('minute')}","Convert the incoming millisecond to match anything on or before the minute in" +
+        " which the millisecond falls"})
+    public DateRangeOnOrBefore(String precision) {
+        this.precision = DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+    }
+
+    @Override
+    public DateRange apply(long value) {
+        Date date = new Date(value);
+        DateRange.DateRangeBound lower = DateRange.DateRangeBound.UNBOUNDED;
+        DateRange.DateRangeBound upper = DateRange.DateRangeBound.upperBound(date,precision);
+        DateRange dateRange = new DateRange(lower, upper);
+        return dateRange;
+    }
+}

activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/to_daterange/DateRangeParser.java

@@ -0,0 +1,36 @@
+package io.nosqlbench.activitytype.cql.datamappers.functions.to_daterange;
+
+import com.datastax.driver.dse.search.DateRange;
+import io.nosqlbench.virtdata.annotations.Example;
+import io.nosqlbench.virtdata.annotations.ThreadSafeMapper;
+
+import java.text.ParseException;
+import java.util.Date;
+import java.util.function.Function;
+
+/**
+ * Parses the DateRange format according to <A HREF="https://lucene.apache.org/solr/guide/6_6/working-with-dates
+ * .html#WorkingwithDates-DateRangeFormatting">Date Range Formatting</A>.
+ * When possible it is more efficient to use the other DateRange methods since they do not require parsing.
+ */
+@ThreadSafeMapper
+public class DateRangeParser implements Function<String, DateRange> {
+
+    private final DateRange.DateRangeBound.Precision precision;
+
+    @Example({"DateRangeParser()}","Convert inputs like '[1970-01-01T00:00:00 TO 1970-01-01T00:00:00]' into " +
+        "DateRanges" +
+        " "})
+    public DateRangeParser(String precision) {
+        this.precision = DateRange.DateRangeBound.Precision.valueOf(precision.toUpperCase());
+    }
+
+    @Override
+    public DateRange apply(String value) {
+        try {
+            return DateRange.parse(value);
+        } catch (ParseException e) {
+            throw new RuntimeException("unable to parse date rage input '" + value + "': error:" + e.getMessage());
+        }
+    }
+}

activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/statements/core/YamlCQLStatementLoader.java

@@ -1,7 +1,7 @@
 package io.nosqlbench.activitytype.cql.statements.core;
 
 import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.TypeDescription;
@@ -32,7 +32,7 @@ public class YamlCQLStatementLoader {
 
     public AvailableCQLStatements load(String fromPath, String... searchPaths) {
 
-        InputStream stream = NosqlBenchFiles.findRequiredStreamOrFile(fromPath,
+        InputStream stream = NBFiles.findRequiredStreamOrFile(fromPath,
                 "yaml", searchPaths);
         String data = "";
         try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream))) {

activitytype-cql/src/test/java/io/nosqlbench/activitytype/cql/datamappers/functions/to_daterange/DateRangeFuncTest.java

@@ -0,0 +1,59 @@
+package io.nosqlbench.activitytype.cql.datamappers.functions.to_daterange;
+
+import org.junit.Test;
+
+import java.util.function.Function;
+import java.util.function.LongFunction;
+import java.util.function.LongUnaryOperator;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DateRangeFuncTest {
+
+    @Test
+    public void testDateRangeFuncs() {
+        LongFunction<Long> lf1 = value -> value;
+        DateRangeFunc function = new DateRangeFunc("second", lf1, lf1);
+
+        assertThat(function.apply(42L).toString())
+            .isEqualTo("[1970-01-01T00:00:00 TO 1970-01-01T00:00:00]");
+        assertThat(function.apply(42000L).toString())
+            .isEqualTo("[1970-01-01T00:00:42 TO 1970-01-01T00:00:42]");
+        assertThat(function.apply(42000000L).toString())
+            .isEqualTo("[1970-01-01T11:40:00 TO 1970-01-01T11:40:00]");
+        assertThat(function.apply(42000000000L).toString())
+            .isEqualTo("[1971-05-02T02:40:00 TO 1971-05-02T02:40:00]");
+        assertThat(function.apply(42000000000000L).toString())
+            .isEqualTo("[3300-12-05T02:40:00 TO 3300-12-05T02:40:00]");
+
+        LongUnaryOperator lf2 = value -> value;
+
+        function = new DateRangeFunc("second", lf2, lf2);
+        assertThat(function.apply(42L).toString())
+            .isEqualTo("[1970-01-01T00:00:00 TO 1970-01-01T00:00:00]");
+        assertThat(function.apply(42000L).toString())
+            .isEqualTo("[1970-01-01T00:00:42 TO 1970-01-01T00:00:42]");
+        assertThat(function.apply(42000000L).toString())
+            .isEqualTo("[1970-01-01T11:40:00 TO 1970-01-01T11:40:00]");
+        assertThat(function.apply(42000000000L).toString())
+            .isEqualTo("[1971-05-02T02:40:00 TO 1971-05-02T02:40:00]");
+        assertThat(function.apply(42000000000000L).toString())
+            .isEqualTo("[3300-12-05T02:40:00 TO 3300-12-05T02:40:00]");
+
+        Function<Long,Long> lf3 = value -> value;
+
+        function = new DateRangeFunc("second", lf3, lf3);
+        assertThat(function.apply(42L).toString())
+            .isEqualTo("[1970-01-01T00:00:00 TO 1970-01-01T00:00:00]");
+        assertThat(function.apply(42000L).toString())
+            .isEqualTo("[1970-01-01T00:00:42 TO 1970-01-01T00:00:42]");
+        assertThat(function.apply(42000000L).toString())
+            .isEqualTo("[1970-01-01T11:40:00 TO 1970-01-01T11:40:00]");
+        assertThat(function.apply(42000000000L).toString())
+            .isEqualTo("[1971-05-02T02:40:00 TO 1971-05-02T02:40:00]");
+        assertThat(function.apply(42000000000000L).toString())
+            .isEqualTo("[3300-12-05T02:40:00 TO 3300-12-05T02:40:00]");
+
+    }
+
+}

docsys/src/test/java/io/nosqlbench/docsys/core/PathWalkerTest.java

@@ -0,0 +1,7 @@
+package io.nosqlbench.docsys.core;
+
+import static org.junit.Assert.*;
+
+public class PathWalkerTest {
+
+}

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java

@@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml;
 
 import io.nosqlbench.engine.api.activityconfig.snakecharmer.SnakeYamlCharmer;
 import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import org.slf4j.Logger;
 import org.yaml.snakeyaml.TypeDescription;
 import org.yaml.snakeyaml.Yaml;
@@ -27,7 +27,6 @@ import org.yaml.snakeyaml.Yaml;
 import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -56,7 +55,7 @@ public class RawYamlStatementLoader {
     }
 
     protected String loadRawFile(Logger logger, String fromPath, String... searchPaths) {
-        InputStream stream = NosqlBenchFiles.findRequiredStreamOrFile(fromPath, "yaml", searchPaths);
+        InputStream stream = NBFiles.findRequiredStreamOrFile(fromPath, "yaml", searchPaths);
         try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream))) {
             return buffer.lines().collect(Collectors.joining("\n"));
         } catch (Exception e) {

engine-api/src/main/java/io/nosqlbench/engine/api/util/NBFiles.java

@@ -34,9 +34,9 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-public class NosqlBenchFiles {
+public class NBFiles {
 
-    private final static Logger logger = LoggerFactory.getLogger(NosqlBenchFiles.class);
+    private final static Logger logger = LoggerFactory.getLogger(NBFiles.class);
     private static Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
     private static Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
 
@@ -73,7 +73,15 @@ public class NosqlBenchFiles {
         return Optional.empty();
     }
 
-    public static Optional<Path> findOptionalPath(String basename, String extension, String... searchPaths) {
+    /**
+     * Search for the path
+     * @param basename Basename of path, with or without extension
+     * @param extension The extension of the filename
+     * @param searchWithin If enabled, all searchPaths are traversed, looking for a matching suffix pattern.
+     * @param searchPaths Additional places to look for the path suffix
+     * @return An optional path
+     */
+    public static Optional<Path> findOptionalPath(String basename, String extension, boolean searchWithin, String... searchPaths) {
 
         boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
         String filename = basename + (needsExtension ? "." + extension : "");
@@ -95,6 +103,12 @@ public class NosqlBenchFiles {
             }
         }
 
+        if (searchWithin) {
+            throw new RuntimeException("not implemented");
+//            for (String searchPath : searchPaths) {
+//                NBPathWalker.findEndMatching(Path.of(searchPath), Path.of(filename));
+//            }
+        }
         return Optional.empty();
     }
 
@@ -127,7 +141,7 @@ public class NosqlBenchFiles {
         }
 
         // Classpath
-        ClassLoader classLoader = NosqlBenchFiles.class.getClassLoader();
+        ClassLoader classLoader = NBFiles.class.getClassLoader();
         InputStream stream = classLoader.getResourceAsStream(path);
         if (stream != null) {
             return Optional.of(stream);

engine-api/src/main/java/io/nosqlbench/engine/api/util/NBPathWalker.java

@@ -0,0 +1,125 @@
+package io.nosqlbench.engine.api.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.ArrayList;
+import java.util.List;
+
+public class NBPathWalker {
+    private final static Logger logger = LogManager.getLogger(NBPathWalker.class);
+
+    public static void walk(Path p, PathVisitor v) {
+        walk(p, v, NBPathWalker.WALK_ALL);
+    }
+
+    public static List<Path> findAll(Path p) {
+        Collect fileCollector = new Collect(true, false);
+        walk(p, fileCollector);
+        return fileCollector.get();
+    }
+    public static List<Path> findEndMatching(Path p, Path endingName) {
+        Collect fileCollector = new Collect(true, false);
+        MatchEnding ending = new MatchEnding(endingName.toString());
+        walk(p, fileCollector, ending);
+        return fileCollector.get();
+    }
+
+    public static void walk(Path p, PathVisitor v, DirectoryStream.Filter<Path> filter) {
+        try {
+            FileSystemProvider provider = p.getFileSystem().provider();
+            DirectoryStream<Path> paths = provider.newDirectoryStream(p, (Path r) -> true);
+            List<Path> pathlist = new ArrayList<>();
+            for (Path path : paths) {
+                pathlist.add(path);
+            }
+            for (Path path : pathlist) {
+                if (path.getFileSystem().provider().readAttributes(path, BasicFileAttributes.class).isDirectory()) {
+                    v.preVisitDir(path);
+                    walk(path, v, filter);
+                    v.postVisitDir(path);
+                } else if (filter.accept(path)) {
+                    v.preVisitFile(path);
+                    v.visit(path);
+                    v.postVisitFile(path);
+
+                } else {
+                    logger.error("error");
+                }
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public interface PathVisitor {
+        void visit(Path p);
+
+        default void preVisitFile(Path path) {
+        }
+
+        default void postVisitFile(Path path) {
+        }
+
+        default void preVisitDir(Path path) {
+        }
+
+        default void postVisitDir(Path path) {
+        }
+    }
+
+    public static DirectoryStream.Filter<Path> WALK_ALL = entry -> true;
+
+    public static class MatchEnding implements DirectoryStream.Filter<Path> {
+        private final String regex;
+
+        public MatchEnding(String pathEnd) {
+            this.regex = pathEnd;
+        }
+
+        @Override
+        public boolean accept(Path entry) throws IOException {
+            return entry.toString().endsWith(regex);
+        }
+    }
+
+    public static class Collect implements PathVisitor {
+        private final List<Path> listing = new ArrayList<>();
+        private final boolean collectFiles;
+        private final boolean collectDirectories;
+
+        public Collect(boolean collectFiles, boolean collectDirectories) {
+
+            this.collectFiles = collectFiles;
+            this.collectDirectories = collectDirectories;
+        }
+
+        public List<Path> get() {
+            return listing;
+        }
+
+        @Override
+        public void visit(Path p) {
+        }
+
+        @Override
+        public void preVisitFile(Path path) {
+            if (this.collectFiles) {
+                listing.add(path);
+            }
+        }
+
+        @Override
+        public void preVisitDir(Path path) {
+            if (this.collectDirectories) {
+                listing.add(path);
+            }
+        }
+    }
+
+}

engine-api/src/test/java/io/nosqlbench/engine/api/util/NBFilesTest.java

@@ -25,11 +25,11 @@ import java.util.Optional;
 import static org.assertj.core.api.Assertions.assertThat;
 
 @Test
-public class NosqlBenchFilesTest {
+public class NBFilesTest {
 
     @Test
     public void testNestedClasspathLoading() {
-        Optional<InputStream> optionalStreamOrFile = NosqlBenchFiles.findOptionalStreamOrFile("nested/testfile", "txt", "activities");
+        Optional<InputStream> optionalStreamOrFile = NBFiles.findOptionalStreamOrFile("nested/testfile", "txt", "activities");
         assertThat(optionalStreamOrFile).isPresent();
     }
 
@@ -46,4 +46,4 @@ public class NosqlBenchFilesTest {
 //        Optional<InputStream> inputStream = NosqlBenchFiles.getInputStream(url);
 //        assertThat(inputStream).isPresent();
 //    }
-}
+}

engine-api/src/test/java/io/nosqlbench/engine/api/util/NBPathWalkerTest.java

@@ -0,0 +1,21 @@
+package io.nosqlbench.engine.api.util;
+
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class NBPathWalkerTest {
+
+//    @Test
+//    public void testBasicPathmatching() {
+//        List<Path> found = NBPathWalker.findEndMatching(
+//            Path.of("testdocs"),
+//            Path.of("identity.yaml")
+//        );
+//
+//        assertThat(found).containsExactly(Path.of("flsd"));
+//    }
+}

engine-api/src/test/resources/testdocs/nested1/identity.yaml

@@ -0,0 +1,2 @@
+bindings:
+  i: Identity();

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -6,7 +6,7 @@ import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogIm
 import io.nosqlbench.engine.api.activityapi.input.InputType;
 import io.nosqlbench.engine.api.activityapi.output.OutputType;
 import io.nosqlbench.engine.api.exceptions.BasicError;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import io.nosqlbench.engine.core.MarkdownDocInfo;
 import io.nosqlbench.engine.core.ScenarioLogger;
 import io.nosqlbench.engine.core.ScenariosResults;
@@ -233,8 +233,8 @@ public class NBCLI {
     }
 
     public void printWorkloads() {
-        List<NosqlBenchFiles.WorkloadDesc> workloads = NosqlBenchFiles.getWorkloadsWithScenarioScripts();
+        List<NBFiles.WorkloadDesc> workloads = NBFiles.getWorkloadsWithScenarioScripts();
-        for (NosqlBenchFiles.WorkloadDesc workload : workloads) {
+        for (NBFiles.WorkloadDesc workload : workloads) {
             System.out.println("\n# from: "+ workload.getYamlPath());
             List<String> scenarioList = workload.getScenarioNames();
             String workloadName = workload.getYamlPath().replaceAll("\\.yaml", "") ;

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -2,7 +2,7 @@ package io.nosqlbench.engine.cli;
 
 import ch.qos.logback.classic.Level;
 import io.nosqlbench.engine.api.metrics.IndicatorMode;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import io.nosqlbench.engine.api.util.Unit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -303,7 +303,7 @@ public class NBCLIOptions {
                     break;
                 default:
                     Optional<InputStream> optionalScript =
-                        NosqlBenchFiles.findOptionalStreamOrFile(word, "js", "scripts/auto");
+                        NBFiles.findOptionalStreamOrFile(word, "js", "scripts/auto");
                     //Script
                     if (optionalScript.isPresent()) {
                         arglist.removeFirst();

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java

@@ -4,7 +4,7 @@ import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import io.nosqlbench.engine.api.exceptions.BasicError;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import io.nosqlbench.engine.api.util.StrInterpolator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,14 +23,14 @@ public class NBCLIScenarioParser {
     private final static Logger logger = LoggerFactory.getLogger(NBCLIScenarioParser.class);
 
     public static boolean isFoundWorkload(String word) {
-        Optional<Path> workloadPath = NosqlBenchFiles.findOptionalPath(word, "yaml", "activities");
+        Optional<Path> workloadPath = NBFiles.findOptionalPath(word, "yaml", false, "activities");
         return workloadPath.isPresent();
     }
 
     public static void parseScenarioCommand(LinkedList<String> arglist) {
 
         String workloadName = arglist.removeFirst();
-        Optional<Path> workloadPathSearch = NosqlBenchFiles.findOptionalPath(workloadName, "yaml", "activities");
+        Optional<Path> workloadPathSearch = NBFiles.findOptionalPath(workloadName, "yaml", false, "activities");
         Path workloadPath = workloadPathSearch.orElseThrow();
 
         List<String> scenarioNames = new ArrayList<>();

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScriptAssembly.java

@@ -1,7 +1,7 @@
 package io.nosqlbench.engine.cli;
 
 import io.nosqlbench.engine.api.activityimpl.ActivityDef;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import io.nosqlbench.engine.api.util.StrInterpolator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -81,7 +81,7 @@ public class NBCLIScriptAssembly {
         } catch (IOException ignored) {
         }
 
-        InputStream resourceAsStream = NosqlBenchFiles.findRequiredStreamOrFile(cmd.getCmdSpec(), "js", "scripts");
+        InputStream resourceAsStream = NBFiles.findRequiredStreamOrFile(cmd.getCmdSpec(), "js", "scripts");
 
         try (BufferedReader buffer = new BufferedReader(new InputStreamReader(resourceAsStream))) {
             scriptData = buffer.lines().collect(Collectors.joining("\n"));

engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java

@@ -12,7 +12,7 @@ import com.github.dockerjava.api.model.ContainerNetworkSettings;
 import com.github.dockerjava.api.model.Frame;
 import com.github.dockerjava.core.async.ResultCallbackTemplate;
 import com.github.dockerjava.core.command.LogContainerResultCallback;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -147,7 +147,7 @@ public class DockerMetricsManager {
     }
 
     private void setupPromFiles(String ip) {
-        String datasource = NosqlBenchFiles.readFile("docker/prometheus/prometheus.yml");
+        String datasource = NBFiles.readFile("docker/prometheus/prometheus.yml");
 
         if (ip == null) {
             logger.error("IP for graphite container not found");

engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java

@@ -1,7 +1,7 @@
 package io.nosqlbench.engine.docker;
 
 import io.nosqlbench.engine.api.exceptions.BasicError;
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,7 +36,7 @@ public class RestHelper {
 
         if (path !=null) {
             logger.debug("POSTing " + path + " to " + url);
-            String dashboard = NosqlBenchFiles.readFile(path);
+            String dashboard = NBFiles.readFile(path);
             logger.debug("length of content for " + path + " is " + dashboard.length());
             builder = builder.POST(HttpRequest.BodyPublishers.ofString(dashboard));
             builder.setHeader("Content-Type", "application/json");

engine-docs/src/main/resources/docs-for-nb/basics/activity_parameters.md

@@ -86,15 +86,13 @@ _default value_ : For now, the default is simply *1*. Users must be
 aware of this setting and adjust it to a reasonable value for their
 workloads.
 
-:::info The threads parameter will work slightly differently for
+:::info
-activities using the async parameter. For example, when `async=500` is
+The threads parameter will work slightly differently for activities using the async parameter. For example, when
-provided, then the number of async operations is split between all
+`async=500` is provided, then the number of async operations is split between all configured threads, and each thread
-configured threads, and each thread will juggle a number of in-flight
+will juggle a number of in-flight operations asynchronously. Without the async parameter, threads determines the logical
-operations asynchronously. Without the async parameter, threads
+concurrency level of nosqlbench in the classic 'request-per-thread' mode. Neither mode is strictly correct, and both
-determines the logical concurrency level of nosqlbench in the classic
+modes can be used for more accurate testing depending on the constraints of your environment.
-'request-per-thread' mode. Neither mode is strictly correct, and both
+:::
-modes can be used for more accurate testing depending on the constraints
-of your environment. :::
 
 A good rule of thumb for setting threads for maximum effect is to set it
 relatively high, such as 10XvCPU when running synchronous workloads

engine-docs/src/main/resources/docs-for-nb/basics/parameter_types.md

@@ -5,13 +5,13 @@ weight: 03
 
 # Parameter Types
 
-To configure an nosqlbench activity to do something meaningful, you have to provide parameters to it. This can occur in
+To configure a NoSQLBench activity to do something meaningful, you have to provide parameters to it. This can occur in
-one of several ways. This section is a guide on nosqlbench parameters, how they layer together, and when to use one form
+one of several ways. This section is a guide on NoSQLBench parameters, how they layer together, and when to use one form
 over another.
 
-The command line is used to configure both the overall nosqlbench runtime (logging, etc) as well as the individual
+The command line is used to configure both the overall runtime (logging, etc) as well as the individual activities and
-activities and scripts. Global nosqlbench options can be distinguished from scenario commands and their parameters
+scripts. Global options can be distinguished from scenario commands and their parameters because global options always
-because because global options always start with a single or --double-hyphen.
+start with a -single or --double-hyphen.
 
 ## Activity Parameters
 
@@ -20,21 +20,20 @@ follow a command, such as `run` or `start`, for example. Scenario commands are a
 hyphens. Every command-line argument that follows a scenario command in the form of `<name>=<value>` is a parameter to
 that command.
 
-Activity parameters can be provided by the nosqlbench core runtime or they can be provided by the activity type. All of
+Activity parameters can be provided by the core runtime or they can be provided by the activity type. It's not important
-the params are usable to configure an activity together. It's not important where they are provided from so long as you
+where they are provided from so long as you know what they do for your workloads, how to configure them, and where to
-know what they do for your workloads, how to configure them, and where to find the docs.
+find the docs. Core parameters are documented
 
-*Core* Activity Parameters are those provided by the core runtime. They are part of the core API and used by every
+*Core* Parameters are those provided by the core runtime. They are part of the core API and used by every
 activity type. Core activity params include type*, *alias*, and *threads*, for example. These parameters are explained
 individually under the next section.
 
-*Custom* Activity Parameters are those provided by an activity type. These parameters are documented for each activity
+*Driver* Parameters are those provided by an activity type. These parameters are documented for each activity type. You
-type. You can see them by running `nosqlbench help <activity type>`.
+can see them by running `nosqlbench help <activity type>`.
 
-Activity type parameters may be dynamic. *Dynamic* Activity Parameters are parameters which may be changed while an
+Driver parameters may be dynamic. *Dynamic* Activity Parameters are parameters which may be changed while an activity is
-activity is running. This means that scenario scripting logic may change some variables while an activity is running,
+running. This means that scenario scripting logic may change some variables while an activity is running, and that the
-and that the runtime should dynamically adjust to match. Dynamic parameters are mainly used in more advanced scripting
+runtime should dynamically adjust to match. Dynamic parameters are mainly used in more advanced scripting scenarios.
-scenarios.
 
 Parameters that are dynamic should be documented as such in the respective activity type's help page.
 
@@ -61,7 +60,7 @@ line or script, the parameters are resolved in the following order:
 
 ## Statement Parameters
 
-Some activities make use of a parameters for statements. These are called _statement parameters_ and are completely
+Some activities make use of parameters for statements. These are called _statement parameters_ and are completely
 different than _activity parameters_. Statement parameters in a YAML allow you to affect *how* a statement is used in a
 workload. Just as with activity level parameters, statement parameters may be supported by the core runtime or by an
 activity type. These are also documented in the respective activity type's documentation included in the 'Activity

engine-extensions/src/main/java/io/nosqlbench/engine/extensions/files/FileAccess.java

@@ -17,12 +17,12 @@
 
 package io.nosqlbench.engine.extensions.files;
 
-import io.nosqlbench.engine.api.util.NosqlBenchFiles;
+import io.nosqlbench.engine.api.util.NBFiles;
 
 public class FileAccess extends FileAccessPluginData {
 
     public String read(String filepath) {
-        String filedata = NosqlBenchFiles.readFile(filepath);
+        String filedata = NBFiles.readFile(filepath);
         return filedata;
 
     }