IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #358 from ivansenic/ise-mongodb-crud

Browse Source 
added mongoDB CRUD scenarios
This commit is contained in:
Jonathan Shook 2021-09-22 10:52:01 -05:00 committed by GitHub
commit 99982ce74d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 344 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
driver-http/src/main/resources/activities/documents-api/http-docsapi-crud-basic.yaml
View File

@ -7,11 +7,11 @@ description: |
scenarios:
default:
schema: run driver=http tags==phase:schema threads==1 cycles==UNDEF
write: run driver=http tags==phase:main,type:write cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
read: run driver=http tags==phase:main,type:read cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
update: run driver=http tags==phase:main,type:update cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
delete: run driver=http tags==phase:main,type:delete cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
schema: run driver=http tags==phase:schema threads==1 cycles==UNDEF
write: run driver=http tags==phase:main,type:write cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
read: run driver=http tags==phase:main,type:read cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
update: run driver=http tags==phase:main,type:update cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
delete: run driver=http tags==phase:main,type:delete cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
bindings:
# To enable an optional weighted set of hosts in place of a load balancer
@ -31,7 +31,7 @@ bindings:
gender: WeightedStrings('M:10;F:10;O:1')
full_name: FullNames()
married: ModuloToBoolean()
city: CSVSampler('city','n/a','name','data/simplemaps/uszips.csv')
city: Cities()
country_code: CountryCodes()
lat: Uniform(-180d, 180d)
lng: Hash() -> long; Uniform(-180d, 180d)
@ -129,7 +129,7 @@ blocks:
phase: main
type: update
statements:
- write-document: PUT <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_basic>>/collections/<<table:docs_collection>>/{random_key}
- update-document: PUT <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_basic>>/collections/<<table:docs_collection>>/{random_key}
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
@ -166,7 +166,7 @@ blocks:
phase: main
type: delete
statements:
- write-document: DELETE <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_basic>>/collections/<<table:docs_collection>>/{seq_key}
- delete-document: DELETE <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_basic>>/collections/<<table:docs_collection>>/{seq_key}
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"

14
driver-http/src/main/resources/activities/documents-api/http-docsapi-crud-dataset.yaml
View File

@ -7,11 +7,11 @@ description: |
scenarios:
default:
schema: run driver=http tags==phase:schema threads==1 cycles==UNDEF
write: run driver=http tags==phase:main,type:write cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
read: run driver=http tags==phase:main,type:read cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
update: run driver=http tags==phase:main,type:update cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
delete: run driver=http tags==phase:main,type:delete cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
schema: run driver=http tags==phase:schema threads==1 cycles==UNDEF
write: run driver=http tags==phase:main,type:write cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
read: run driver=http tags==phase:main,type:read cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
update: run driver=http tags==phase:main,type:update cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
delete: run driver=http tags==phase:main,type:delete cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
bindings:
# To enable an optional weighted set of hosts in place of a load balancer
@ -97,7 +97,7 @@ blocks:
phase: main
type: update
statements:
- write-document: PUT <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_dataset>>/collections/<<table:docs_collection>>/{random_key}
- update-document: PUT <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_dataset>>/collections/<<table:docs_collection>>/{random_key}
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"
@ -113,7 +113,7 @@ blocks:
phase: main
type: delete
statements:
- write-document: DELETE <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_dataset>>/collections/<<table:docs_collection>>/{seq_key}
- update-document: DELETE <<protocol:http>>://{weighted_hosts}:<<stargate_port:8082>><<path_prefix:>>/v2/namespaces/<<keyspace:docs_crud_dataset>>/collections/<<table:docs_collection>>/{seq_key}
Accept: "application/json"
X-Cassandra-Request-Id: "{request_id}"
X-Cassandra-Token: "<<auth_token:my_auth_token>>"

14
driver-mongodb/src/main/resources/activities/mongodb-crud-basic.md Normal file
View File

@ -0,0 +1,14 @@
---
title: mongoDB CRUD Basic
weight: 1
---
## Description
The mongoDB CRUD Basic workflow emulates CRUD operations for the mongoDB using generated JSON documents.
It's a counterpart of the Stargate's Documents API CRUD Basic workflow.
Please refer to [http-docsapi-crud-basic.md](../../../../../driver-http/src/main/resources/activities/documents-api/http-docsapi-crud-basic.md) for the general workflow design details.
## Indexing
To simulate a realistic situation as much as possible, this workflow creates 3 additional indexes (apart from `_id`) for the collection where documents are stored.

183
driver-mongodb/src/main/resources/activities/mongodb-crud-basic.yaml Normal file
View File

@ -0,0 +1,183 @@
# nb -v run driver=mongodb yaml=mongodb-crud-basic tags=phase:schema connection=mongodb://127.0.0.1 database=testdb
description: |
This workload emulates CRUD operations for the mongoDB.
It generates a simple JSON document to be used for writes and updates.
It's a counterpart of the Stargate's Documents API CRUD Basic workflow.
scenarios:
default:
schema: run driver=mongodb tags==phase:schema threads==1 cycles==UNDEF
write: run driver=mongodb tags==phase:main,type:write cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
read: run driver=mongodb tags==phase:main,type:read cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
update: run driver=mongodb tags==phase:main,type:update cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
delete: run driver=mongodb tags==phase:main,type:delete cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
bindings:
seq_key: Mod(<<docscount:10000000>>); ToString() -> String
random_key: Uniform(0,<<docscount:10000000>>); ToString() -> String
user_id: ToHashedUUID(); ToString() -> String
created_on: Uniform(1262304000,1577836800) -> long
gender: WeightedStrings('M:10;F:10;O:1')
full_name: FullNames()
married: ModuloToBoolean()
city: Cities()
country_code: CountryCodes()
lat: Uniform(-180d, 180d)
lng: Hash() -> long; Uniform(-180d, 180d)
friend_id: Add(-1); ToHashedUUID(); ToString() -> String
blocks:
- tags:
phase: schema
statements:
- dummy-insert: |
{
insert: "<<collection:crud_basic>>",
documents: [ { _id: "dummyyyy" } ]
}
- drop-collection: |
{
drop: "<<collection:crud_basic>>"
}
tags:
name: drop-collection
- create-collection: |
{
create: "<<collection:crud_basic>>"
}
tags:
name: create-collection
- create-indexes: |
{
createIndexes: "<<collection:crud_basic>>",
indexes: [
{
key: { user_id: 1 },
name: "user_id_idx",
unique: true
},
{
key: { created_on: 1 },
name: "created_on_idx"
},
{
key: { gender: 1 },
name: "gender_idx"
}
]
}
tags:
name: create-indexes
- name: main-write
tags:
phase: main
type: write
statements:
- write-document: |
{
insert: "<<collection:crud_basic>>",
documents: [
{
"_id": "{seq_key}",
"user_id": "{user_id}",
"created_on": {created_on},
"gender": "{gender}",
"full_name": "{full_name}",
"married": {married},
"address": {
"primary": {
"city": "{city}",
"cc": "{country_code}"
},
"secondary": {}
},
"coordinates": [
{lat},
{lng}
],
"children": [],
"friends": [
"{friend_id}"
],
"debt": null
}
]
}
tags:
name: write-document
- name: main-read
tags:
phase: main
type: read
statements:
- read-document: |
{
find: "<<collection:crud_basic>>",
filter: { _id: "{random_key}" }
}
tags:
name: read-document
- name: main-update
tags:
phase: main
type: update
statements:
- update-document: |
{
update: "<<collection:crud_basic>>",
updates: [
{
q: { _id: "{random_key}" },
u: {
"_id": "{seq_key}",
"user_id": "{user_id}",
"created_on": {created_on},
"gender": "{gender}",
"full_name": "{full_name}",
"married": {married},
"address": {
"primary": {
"city": "{city}",
"cc": "{country_code}"
},
"secondary": {}
},
"coordinates": [
{lat},
{lng}
],
"children": [],
"friends": [
"{friend_id}"
],
"debt": null
}
}
]
}
tags:
name: update-document
- name: main-delete
tags:
phase: main
type: delete
statements:
- delete-document: |
{
delete: "<<collection:crud_basic>>",
deletes: [
{
q: { _id: "{seq_key}" },
limit: 1
}
]
}

18
driver-mongodb/src/main/resources/activities/mongodb-crud-dataset.md Normal file
View File

@ -0,0 +1,18 @@
---
title: mongoDB CRUD Dataset
weight: 2
---
## Description
The mongoDB CRUD Dataset workflow emulates CRUD operations for the mongoDB using JSON documents from an external dataset.
It's a counterpart of the Stargate's Documents API CRUD Dataset workflow.
Please refer to [http-docsapi-crud-dataset.md](../../../../../driver-http/src/main/resources/activities/documents-api/http-docsapi-crud-dataset.md) for the general workflow design details.
## Indexing
To simulate a realistic situation as much as possible, this workflow allows creation of the indexes using the parameter:
* `indexes` - Specifies the indexes to create. Each document in the array specifies a separate index. Corresponds to the `indexes` field in the [mongoDB *createIndexes* command](https://docs.mongodb.com/manual/reference/command/createIndexes/#mongodb-dbcommand-dbcmd.createIndexes).
If parameter `indexes` is not specify, a dummy sparse index will be created.

114
driver-mongodb/src/main/resources/activities/mongodb-crud-dataset.yaml Normal file
View File

@ -0,0 +1,114 @@
# nb -v run driver=mongodb yaml=mongodb-crud-dataset tags=phase:schema connection=mongodb://127.0.0.1 database=testdb dataset_file=path/to/data.json
description: |
This workload emulates CRUD operations for the mongoDB.
It requires a data set file, where each line is a single JSON document to be used for writes and updates.
It's a counterpart of the Stargate's Documents API CRUD Dataset workflow.
scenarios:
default:
schema: run driver=mongodb tags==phase:schema threads==1 cycles==UNDEF
write: run driver=mongodb tags==phase:main,type:write cycles===TEMPLATE(write-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
read: run driver=mongodb tags==phase:main,type:read cycles===TEMPLATE(read-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
update: run driver=mongodb tags==phase:main,type:update cycles===TEMPLATE(update-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
delete: run driver=mongodb tags==phase:main,type:delete cycles===TEMPLATE(delete-cycles,TEMPLATE(docscount,10000000)) threads=auto errors=timer,warn
bindings:
seq_key: Mod(<<docscount:10000000>>); ToString() -> String
random_key: Uniform(0,<<docscount:10000000>>); ToString() -> String
blocks:
- tags:
phase: schema
statements:
- dummy-insert: |
{
insert: "<<collection:crud_dataset>>",
documents: [ { _id: "dummyyyy" } ]
}
- drop-collection: |
{
drop: "<<collection:crud_dataset>>"
}
tags:
name: drop-collection
- create-collection: |
{
create: "<<collection:crud_dataset>>"
}
tags:
name: create-collection
- create-indexes: |
{
createIndexes: "<<collection:crud_dataset>>",
indexes: <<indexes:[ { key: { dummy : 1 }, name: "dummy_idx", sparse: true } ]>>
}
tags:
name: create-indexes
- name: main-write
tags:
phase: main
type: write
statements:
- write-document: |
{
insert: "<<collection:crud_dataset>>",
documents: [ { "_id": "{seq_key}", {document_json_without_id} ]
}
tags:
name: write-document
bindings:
document_json_without_id: ModuloLineToString('<<dataset_file>>'); ReplaceRegex('^\{', '')
- name: main-read
tags:
phase: main
type: read
statements:
- read-document: |
{
find: "<<collection:crud_dataset>>",
filter: { _id: "{random_key}" }
}
tags:
name: read-document
- name: main-update
tags:
phase: main
type: update
statements:
- update-document: |
{
update: "<<collection:crud_dataset>>",
updates: [
{
q: { _id: "{random_key}" },
u: { "_id": "{random_key}", {document_json_without_id}
}
]
}
tags:
name: update-document
bindings:
document_json_without_id: ModuloLineToString('<<dataset_file>>'); ReplaceRegex('^\{', '')
- name: main-delete
tags:
phase: main
type: delete
statements:
- delete-document: |
{
delete: "<<collection:crud_dataset>>",
deletes: [
{
q: { _id: "{seq_key}" },
limit: 1
}
]
}