TextOfFile binding; stargate support; scenario updates

adapter-dynamodb/src/main/resources/activities/baselinesv2/dynamodb-keyvalue2.yaml

@@ -8,7 +8,7 @@ scenarios:
   default:
     schema: run driver=dynamodb tags==block:schema threads==1 cycles==UNDEF
     rampup: run driver=dynamodb tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
-    main: run driver=dynamodb tags=="block:main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
+    main: run driver=dynamodb tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
     read: run driver=dynamodb tags==block:main-read cycles===TEMPLATE(main-cycles,10000000) threads=auto
     write: run driver=dynamodb tags==block:main-write cycles===TEMPLATE(main-cycles,10000000) threads=auto
 

adapter-dynamodb/src/main/resources/activities/baselinesv2/dynamodb-tabular2.yaml

@@ -7,7 +7,7 @@ scenarios:
   schema: run driver=dynamodb tags=block:schema region=us-east-1
   rampup: run driver=dynamodb tags=block:rampup region=us-east-1
   read: run driver=dynamodb tags=block:read region=us-east-1
-  main: run driver=dynamodb tags=block:"main.*" region=us-east-1
+  main: run driver=dynamodb tags=block:"main-*.*" region=us-east-1
   read01: run driver=dynamodb tags='name:.*main-read-01' region=us-east-1
   delete:
     table: run driver=dynamodb tags==block:delete threads==1 cycles==UNDEF

adapter-http/src/main/resources/activities/baselines/http-rest-keyvalue.yaml

@@ -9,8 +9,8 @@ description: |
 scenarios:
   default:
     schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
-    rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    rampup: run driver=http tags==block:"rampup-*.*" cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
-    main: run driver=http tags==block:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
+    main: run driver=http tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
 
 bindings:
   # To enable an optional weighted set of hosts in place of a load balancer
@@ -19,11 +19,13 @@ bindings:
   #   multiple hosts: restapi_host=host1,host2,host3
   #   multiple weighted hosts: restapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<restapi_host:stargate>>')
-  # http request id
+
   request_id: ToHashedUUID(); ToString();
+  request_token: ToString(); TextOfFile("TEMPLATE(stargate_tokenfile,data/stargate_token.txt)")
 
   seq_key: Mod(<<keycount:10000000>>); ToString() -> String
   seq_value: Hash(); Mod(<<valuecount:1000000000>>); ToString() -> String
+
   rw_key: <<keydist:Uniform(0,10000000)->int>>; ToString() -> String
   rw_value: Hash(); <<valdist:Uniform(0,1000000000)->int>>; ToString() -> String
 
@@ -35,7 +37,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -48,7 +50,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables/<<table:keyvalue>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         ok-status: "[2-4][0-9][0-9]"
 
@@ -57,7 +59,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -87,7 +89,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -117,7 +119,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:keyvalue>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -134,7 +136,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:keyvalue>>/{rw_key}
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         ok-status: "[2-4][0-9][0-9]"
 
@@ -147,7 +149,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:keyvalue>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {

adapter-http/src/main/resources/activities/baselines/http-rest-tabular.yaml

@@ -9,9 +9,9 @@ description: |
 
 scenarios:
   default:
-    schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
+    schema: run driver=http tags==block:"schema.*" threads==1 cycles==UNDEF
-    rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    rampup: run driver=http tags==block:"rampup-*.*" cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
-    main: run driver=http tags==block:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
+    main: run driver=http tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
 
 bindings:
   # To enable an optional weighted set of hosts in place of a load balancer
@@ -20,16 +20,19 @@ bindings:
   #   multiple hosts: restapi_host=host1,host2,host3
   #   multiple weighted hosts: restapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<restapi_host:stargate>>')
-  # http request id
   request_id: ToHashedUUID(); ToString();
+  request_token: ToString(); TextOfFile("TEMPLATE(stargate_tokenfile,data/stargate_token.txt)")
+
   # for ramp-up and verify
   part_layout: Div(<<partsize:1000000>>); ToString() -> String
   clust_layout: Mod(<<partsize:1000000>>); ToString() -> String
   data: HashedFileExtractToString('data/lorem_ipsum_full.txt',50,150); URLEncode();
+
   # for read
   limit: Uniform(1,10) -> int
   part_read: Uniform(0,<<partcount:100>>)->int; ToString() -> String
   clust_read: Add(1); Uniform(0,<<partsize:1000000>>)->int; ToString() -> String
+
   # for write
   part_write: Hash(); Uniform(0,<<partcount:100>>)->int; ToString() -> String
   clust_write: Hash(); Add(1); Uniform(0,<<partsize:1000000>>)->int; ToString() -> String
@@ -43,7 +46,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -56,7 +59,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables/<<table:tabular>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         ok-status: "[2-4][0-9][0-9]"
 
@@ -65,7 +68,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -102,7 +105,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:tabular>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -120,7 +123,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:tabular>>/{part_read}&page-size={limit}
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
 
   main-write:
@@ -132,7 +135,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:tabular>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {

adapter-http/src/main/resources/activities/baselines/http-rest-timeseries.yaml

@@ -12,10 +12,10 @@ description: |
 
 scenarios:
   default:
-    schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
+    schema: run driver=cql tags==block:"schema-*.*" threads==1 cycles==UNDEF
     schema-astra: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF
-    rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
+    rampup: run driver=http tags==block:"rampup-*.*" cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
-    main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
+    main: run driver=http tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
 
 bindings:
   # To enable an optional weighted set of hosts in place of a load balancer
@@ -24,8 +24,10 @@ bindings:
   #   multiple hosts: restapi_host=host1,host2,host3
   #   multiple weighted hosts: restapi_host=host1:3,host2:7
   weighted_hosts: WeightedStrings('<<restapi_host:stargate>>')
+
   # http request id
   request_id: ToHashedUUID(); ToString();
+  request_token: ToString(); TextOfFile("TEMPLATE(stargate_tokenfile,data/stargate_token.txt)")
 
   machine_id: Mod(<<sources:10000>>); ToHashedUUID() -> java.util.UUID
   sensor_name: HashedLineToString('data/variable_words.txt')
@@ -42,7 +44,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -55,7 +57,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables/<<table:iot>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         ok-status: "[2-4][0-9][0-9]"
 
@@ -64,7 +66,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -123,7 +125,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:iot>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {
@@ -144,7 +146,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:iot>>?where=URLENCODE[[{"machine_id":{"$eq":"{machine_id}"},"sensor_name":{"$eq":"{sensor_name}"}}]]&page-size=<<limit:10>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
 
   main-write:
@@ -156,7 +158,7 @@ blocks:
         uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:iot>>
         Accept: "application/json"
         X-Cassandra-Request-Id: "{request_id}"
-        X-Cassandra-Token: "<<auth_token:my_auth_token>>"
+        X-Cassandra-Token: "{request_token}"
         Content-Type: "application/json"
         body: |
           {

adapter-http/src/main/resources/data/stargate_token.txt

@@ -0,0 +1 @@
+# <<put-token-here>>

nb-api/src/main/java/io/nosqlbench/api/content/NBIO.java

@@ -76,6 +76,7 @@ public class NBIO implements NBPathsAPI.Facets {
         return Arrays.asList(split);
     }
 
+
     public static CSVParser readFileCSV(String filename, String... searchPaths) {
         return NBIO.readFileDelimCSV(filename, ',', searchPaths);
     }

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/unary_string/TextOfFile.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.library.basics.shared.unary_string;
+
+import io.nosqlbench.api.content.NBIO;
+import io.nosqlbench.api.errors.BasicError;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * Provides a single line of text from a target file provided.
+ */
+@ThreadSafeMapper
+@Categories({Category.general})
+public class TextOfFile implements Function<Object, String> {
+    private static final Logger logger = LogManager.getLogger(TextOfFile.class);
+    private final String text;
+
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+
+    @Example({"TextOfFile()", "Provides the first line of text in the specified file."})
+    public TextOfFile(String targetFile) {
+
+        try {
+            final List<String> lines = NBIO.readLines(targetFile);
+            logger.info("TextOfFile() reading: {}", targetFile);
+            if (lines.isEmpty()) {
+                throw new BasicError(String.format("Unable to locate content for %s", this));
+            }
+            text = lines.get(0);
+        } catch (Exception ex) {
+            throw new BasicError(String.format("Unable to locate file %s: ", targetFile), ex);
+        }
+    }
+
+    @Override
+    public String apply(Object obj) {
+        return text;
+    }
+
+}

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/unary_string/TextOfFileTest.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.nosqlbench.virtdata.library.basics.shared.unary_string;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatException;
+
+class TextOfFileTest {
+
+    private static final String EXPECTED_CONTENTS = "test-data-entry";
+    private static final String NOT_EXPECTED_CONTENTS = "foozy-content";
+    private static final String VALID_PATH = "text-provider-sample.txt";
+    private static final String INVALID_PATH = "not-good.txt";
+    private static final String PLACEHOLDER_APPLY_INPUT = "placeholder-input";
+
+
+    @Test
+    void testValidPathAndContents() {
+        final TextOfFile TextOfFile = new TextOfFile(VALID_PATH);
+        assertThat(TextOfFile.apply(PLACEHOLDER_APPLY_INPUT)).isEqualTo(EXPECTED_CONTENTS);
+    }
+
+    @Test
+    void testInvalidPathAndContents() {
+        final TextOfFile textOfFileValid = new TextOfFile(VALID_PATH);
+        assertThatException().isThrownBy(() -> new TextOfFile(INVALID_PATH));
+        assertThat(textOfFileValid.apply(PLACEHOLDER_APPLY_INPUT)).isNotEqualTo(NOT_EXPECTED_CONTENTS);
+    }
+
+}

virtdata-lib-basics/src/test/resources/data/text-provider-sample.txt

@@ -0,0 +1 @@
+test-data-entry