IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

TextOfFile binding; stargate support; scenario updates

Browse Source
This commit is contained in:
jeffbanks 2023-02-03 15:40:33 -06:00
parent 00d27c38d3
commit 6b2a368d79
10 changed files with 150 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
adapter-dynamodb/src/main/resources/activities/baselinesv2/dynamodb-keyvalue2.yaml
View File

@ -8,7 +8,7 @@ scenarios:
default:
schema: run driver=dynamodb tags==block:schema threads==1 cycles==UNDEF
rampup: run driver=dynamodb tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=dynamodb tags=="block:main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
main: run driver=dynamodb tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
read: run driver=dynamodb tags==block:main-read cycles===TEMPLATE(main-cycles,10000000) threads=auto
write: run driver=dynamodb tags==block:main-write cycles===TEMPLATE(main-cycles,10000000) threads=auto

2
adapter-dynamodb/src/main/resources/activities/baselinesv2/dynamodb-tabular2.yaml
View File

@ -7,7 +7,7 @@ scenarios:
schema: run driver=dynamodb tags=block:schema region=us-east-1
rampup: run driver=dynamodb tags=block:rampup region=us-east-1
read: run driver=dynamodb tags=block:read region=us-east-1
main: run driver=dynamodb tags=block:"main.*" region=us-east-1
main: run driver=dynamodb tags=block:"main-*.*" region=us-east-1
read01: run driver=dynamodb tags='name:.*main-read-01' region=us-east-1
delete:
table: run driver=dynamodb tags==block:delete threads==1 cycles==UNDEF

22
adapter-http/src/main/resources/activities/baselines/http-rest-keyvalue.yaml
View File

@ -9,8 +9,8 @@ description: |
scenarios:
default:
schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=http tags==block:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
rampup: run driver=http tags==block:"rampup-*.*" cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=http tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:
# To enable an optional weighted set of hosts in place of a load balancer
@ -19,11 +19,13 @@ bindings:
# multiple hosts: restapi_host=host1,host2,host3
# multiple weighted hosts: restapi_host=host1:3,host2:7
weighted_hosts: WeightedStrings('<<restapi_host:stargate>>')
# http request id
request_id: ToHashedUUID(); ToString();
request_token: ToString(); TextOfFile("TEMPLATE(stargate_tokenfile,data/stargate_token.txt)")
seq_key: Mod(<<keycount:10000000>>); ToString() -> String
seq_value: Hash(); Mod(<<valuecount:1000000000>>); ToString() -> String
rw_key: <<keydist:Uniform(0,10000000)->int>>; ToString() -> String
rw_value: Hash(); <<valdist:Uniform(0,1000000000)->int>>; ToString() -> String
@ -35,7 +37,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -48,7 +50,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables/<<table:keyvalue>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
ok-status: "[2-4][0-9][0-9]"
@ -57,7 +59,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -87,7 +89,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -117,7 +119,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:keyvalue>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -134,7 +136,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:keyvalue>>/{rw_key}
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
ok-status: "[2-4][0-9][0-9]"
@ -147,7 +149,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:keyvalue>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{

23
adapter-http/src/main/resources/activities/baselines/http-rest-tabular.yaml
View File

@ -9,9 +9,9 @@ description: |
scenarios:
default:
schema: run driver=http tags==block:schema threads==1 cycles==UNDEF
rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=http tags==block:main cycles===TEMPLATE(main-cycles,10000000) threads=auto
schema: run driver=http tags==block:"schema.*" threads==1 cycles==UNDEF
rampup: run driver=http tags==block:"rampup-*.*" cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=http tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:
# To enable an optional weighted set of hosts in place of a load balancer
@ -20,16 +20,19 @@ bindings:
# multiple hosts: restapi_host=host1,host2,host3
# multiple weighted hosts: restapi_host=host1:3,host2:7
weighted_hosts: WeightedStrings('<<restapi_host:stargate>>')
# http request id
request_id: ToHashedUUID(); ToString();
request_token: ToString(); TextOfFile("TEMPLATE(stargate_tokenfile,data/stargate_token.txt)")
# for ramp-up and verify
part_layout: Div(<<partsize:1000000>>); ToString() -> String
clust_layout: Mod(<<partsize:1000000>>); ToString() -> String
data: HashedFileExtractToString('data/lorem_ipsum_full.txt',50,150); URLEncode();
# for read
limit: Uniform(1,10) -> int
part_read: Uniform(0,<<partcount:100>>)->int; ToString() -> String
clust_read: Add(1); Uniform(0,<<partsize:1000000>>)->int; ToString() -> String
# for write
part_write: Hash(); Uniform(0,<<partcount:100>>)->int; ToString() -> String
clust_write: Hash(); Add(1); Uniform(0,<<partsize:1000000>>)->int; ToString() -> String
@ -43,7 +46,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -56,7 +59,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables/<<table:tabular>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
ok-status: "[2-4][0-9][0-9]"
@ -65,7 +68,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -102,7 +105,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:tabular>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -120,7 +123,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:tabular>>/{part_read}&page-size={limit}
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
main-write:
@ -132,7 +135,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:tabular>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{

20
adapter-http/src/main/resources/activities/baselines/http-rest-timeseries.yaml
View File

@ -12,10 +12,10 @@ description: |
scenarios:
default:
schema: run driver=cql tags==block:schema threads==1 cycles==UNDEF
schema: run driver=cql tags==block:"schema-*.*" threads==1 cycles==UNDEF
schema-astra: run driver=cql tags==block:schema-astra threads==1 cycles==UNDEF
rampup: run driver=http tags==block:rampup cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=http tags==block:"main.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
rampup: run driver=http tags==block:"rampup-*.*" cycles===TEMPLATE(rampup-cycles,10000000) threads=auto
main: run driver=http tags==block:"main-*.*" cycles===TEMPLATE(main-cycles,10000000) threads=auto
bindings:
# To enable an optional weighted set of hosts in place of a load balancer
@ -24,8 +24,10 @@ bindings:
# multiple hosts: restapi_host=host1,host2,host3
# multiple weighted hosts: restapi_host=host1:3,host2:7
weighted_hosts: WeightedStrings('<<restapi_host:stargate>>')
# http request id
request_id: ToHashedUUID(); ToString();
request_token: ToString(); TextOfFile("TEMPLATE(stargate_tokenfile,data/stargate_token.txt)")
machine_id: Mod(<<sources:10000>>); ToHashedUUID() -> java.util.UUID
sensor_name: HashedLineToString('data/variable_words.txt')
@ -42,7 +44,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -55,7 +57,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables/<<table:iot>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
ok-status: "[2-4][0-9][0-9]"
@ -64,7 +66,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/schemas/keyspaces/<<keyspace:baselines>>/tables
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -123,7 +125,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:iot>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{
@ -144,7 +146,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:iot>>?where=URLENCODE[[{"machine_id":{"$eq":"{machine_id}"},"sensor_name":{"$eq":"{sensor_name}"}}]]&page-size=<<limit:10>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
main-write:
@ -156,7 +158,7 @@ blocks:
uri: <<protocol:http>>://{weighted_hosts}:<<restapi_port:8082>><<path_prefix:>>/v2/keyspaces/<<keyspace:baselines>>/<<table:iot>>
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
X-Cassandra-Token: "{request_token}"
Content-Type: "application/json"
body: |
{

1
adapter-http/src/main/resources/data/stargate_token.txt Normal file
View File

@ -0,0 +1 @@
# <<put-token-here>>

1
nb-api/src/main/java/io/nosqlbench/api/content/NBIO.java
View File

@ -76,6 +76,7 @@ public class NBIO implements NBPathsAPI.Facets {
return Arrays.asList(split);
}
public static CSVParser readFileCSV(String filename, String... searchPaths) {
return NBIO.readFileDelimCSV(filename, ',', searchPaths);
}

64
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/unary_string/TextOfFile.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.unary_string;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.function.Function;
/**
* Provides a single line of text from a target file provided.
*/
@ThreadSafeMapper
@Categories({Category.general})
public class TextOfFile implements Function<Object, String> {
private static final Logger logger = LogManager.getLogger(TextOfFile.class);
private final String text;
public String toString() {
return getClass().getSimpleName();
}
@Example({"TextOfFile()", "Provides the first line of text in the specified file."})
public TextOfFile(String targetFile) {
try {
final List<String> lines = NBIO.readLines(targetFile);
logger.info("TextOfFile() reading: {}", targetFile);
if (lines.isEmpty()) {
throw new BasicError(String.format("Unable to locate content for %s", this));
}
text = lines.get(0);
} catch (Exception ex) {
throw new BasicError(String.format("Unable to locate file %s: ", targetFile), ex);
}
}
@Override
public String apply(Object obj) {
return text;
}
}

45
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/unary_string/TextOfFileTest.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.unary_string;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatException;
class TextOfFileTest {
private static final String EXPECTED_CONTENTS = "test-data-entry";
private static final String NOT_EXPECTED_CONTENTS = "foozy-content";
private static final String VALID_PATH = "text-provider-sample.txt";
private static final String INVALID_PATH = "not-good.txt";
private static final String PLACEHOLDER_APPLY_INPUT = "placeholder-input";
@Test
void testValidPathAndContents() {
final TextOfFile TextOfFile = new TextOfFile(VALID_PATH);
assertThat(TextOfFile.apply(PLACEHOLDER_APPLY_INPUT)).isEqualTo(EXPECTED_CONTENTS);
}
@Test
void testInvalidPathAndContents() {
final TextOfFile textOfFileValid = new TextOfFile(VALID_PATH);
assertThatException().isThrownBy(() -> new TextOfFile(INVALID_PATH));
assertThat(textOfFileValid.apply(PLACEHOLDER_APPLY_INPUT)).isNotEqualTo(NOT_EXPECTED_CONTENTS);
}
}

1
virtdata-lib-basics/src/test/resources/data/text-provider-sample.txt Normal file
View File

@ -0,0 +1 @@
test-data-entry