From 5977ad7c0fe0d7c85c27b4735bba7e87c98e0e7a Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 27 Jun 2022 23:45:06 -0500 Subject: [PATCH] convert mongodb driver from ActivityType to DriverAdapter --- {driver-mongodb => adapter-mongodb}/pom.xml | 7 +- .../mongodb/core/MongoOpDispenser.java | 72 ++++++++ .../adapter/mongodb/core/MongoSpace.java | 86 +++++++++ .../mongodb/core/MongodbDriverAdapter.java | 49 +++++ .../adapter/mongodb/core/MongodbOpMapper.java | 41 +++++ .../adapter/mongodb/ops/MongoOp.java | 53 ++++++ .../mongodb/ops/MongoOpFailedException.java | 38 ++++ .../activities/mongodb-basic-uuid.yaml | 0 .../resources/activities/mongodb-basic.yaml | 0 .../activities/mongodb-crud-basic.md | 0 .../activities/mongodb-crud-basic.yaml | 0 .../activities/mongodb-crud-dataset.md | 0 .../activities/mongodb-crud-dataset.yaml | 0 .../activities/mongodb-search-basic.md | 0 .../activities/mongodb-search-basic.yaml | 0 .../src/main/resources/mongodb.md | 0 .../driver/mongodb/MongoAction.java | 91 --------- .../driver/mongodb/MongoActivity.java | 173 ------------------ .../driver/mongodb/MongoActivityType.java | 51 ------ .../driver/mongodb/ReadyMongoStatement.java | 51 ------ .../driver/mongodb/MongoActivityTest.java | 49 ----- .../mongodb/ReadyMongoStatementTest.java | 121 ------------ nb5/pom.xml | 6 + pom.xml | 1 + 24 files changed, 349 insertions(+), 540 deletions(-) rename {driver-mongodb => adapter-mongodb}/pom.xml (91%) create mode 100644 adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpDispenser.java create mode 100644 adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoSpace.java create mode 100644 adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbDriverAdapter.java create mode 100644 adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbOpMapper.java create mode 100644 adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOp.java create mode 100644 adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOpFailedException.java rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-basic-uuid.yaml (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-basic.yaml (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-crud-basic.md (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-crud-basic.yaml (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-crud-dataset.md (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-crud-dataset.yaml (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-search-basic.md (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/activities/mongodb-search-basic.yaml (100%) rename {driver-mongodb => adapter-mongodb}/src/main/resources/mongodb.md (100%) delete mode 100644 driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoAction.java delete mode 100644 driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java delete mode 100644 driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivityType.java delete mode 100644 driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java delete mode 100644 driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/MongoActivityTest.java delete mode 100644 driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java diff --git a/driver-mongodb/pom.xml b/adapter-mongodb/pom.xml similarity index 91% rename from driver-mongodb/pom.xml rename to adapter-mongodb/pom.xml index 54014d4f9..9abb4031b 100644 --- a/driver-mongodb/pom.xml +++ b/adapter-mongodb/pom.xml @@ -17,7 +17,7 @@ 4.0.0 - driver-mongodb + adapter-mongodb jar @@ -29,8 +29,7 @@ ${project.artifactId} - An nosqlbench ActivityType (AT) driver module; - MongoDB + An nosqlbench DriverAdapter module for MongoDB @@ -43,7 +42,7 @@ org.mongodb mongodb-driver-sync - 4.5.0 + 4.6.1 diff --git a/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpDispenser.java b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpDispenser.java new file mode 100644 index 000000000..e9178efab --- /dev/null +++ b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoOpDispenser.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.mongodb.core; + +import com.mongodb.ReadPreference; +import io.nosqlbench.adapter.mongodb.ops.MongoOp; +import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.engine.api.templating.ParsedOp; +import org.bson.Document; +import org.bson.conversions.Bson; + +import java.util.Map; +import java.util.function.LongFunction; + +public class MongoOpDispenser extends BaseOpDispenser { + private final LongFunction opFunc; + private final LongFunction mongoOpF; + + public MongoOpDispenser(LongFunction ctxFunc, ParsedOp op) { + super(op); + opFunc = createOpFunc(ctxFunc, op); + this.mongoOpF = createOpFunc(ctxFunc,op); + } + + private LongFunction createOpFunc(LongFunction ctxFunc, ParsedOp op) { + + LongFunction rpstring = op.getAsOptionalFunction("readPreference") + .orElseGet(() -> op.getAsOptionalFunction("read-preference") + .orElse(l -> "primary")); + LongFunction readPreferenceF = l -> ReadPreference.valueOf(rpstring.apply(l)); + + LongFunction payload = op.getAsRequiredFunction("stmt", Object.class); + Object exampleValue = payload.apply(0); + + LongFunction bsonFunc; + if (exampleValue instanceof CharSequence cs) { + bsonFunc = l -> Document.parse(payload.apply(l).toString()); + } else if ( exampleValue instanceof Map map) { + bsonFunc = l -> new Document((Map)payload.apply(l)); + } else { + throw new RuntimeException("You must provide a String or Map for your BSON payload."); + } + + LongFunction databaseNamerF = op.getAsRequiredFunction("database", String.class); + + return l-> new MongoOp( + ctxFunc.apply(l).getClient(), + databaseNamerF.apply(l), + bsonFunc.apply(l) + ); + } + + @Override + public Op apply(long cycle) { + return mongoOpF.apply(cycle); + } +} diff --git a/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoSpace.java b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoSpace.java new file mode 100644 index 000000000..963fdc4dc --- /dev/null +++ b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoSpace.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.mongodb.core; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; +import io.nosqlbench.api.NBNamedElement; +import io.nosqlbench.nb.api.config.standard.ConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.config.standard.Param; +import org.bson.UuidRepresentation; +import org.bson.codecs.UuidCodec; +import org.bson.codecs.configuration.CodecRegistry; + +import static org.bson.codecs.configuration.CodecRegistries.fromCodecs; +import static org.bson.codecs.configuration.CodecRegistries.fromRegistries; + +public class MongoSpace implements NBNamedElement { + private final String name; + private final NBConfiguration cfg; + private final String connectionString; + private final MongoClient client; + private MongoDatabase mongoDatabase; + + public MongoSpace(String name, NBConfiguration cfg) { + this.name = name; + this.cfg = cfg; + this.connectionString = cfg.get("connection",String.class); + this.client = createMongoClient(connectionString); + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(MongoSpace.class) + .add(Param.required("connection", String.class) + .setDescription("The connection string for your MongoDB endpoint")) + .add(Param.required("database", String.class) + .setDescription("The database name to connect to.")) + .asReadOnly(); + } + + @Override + public String getName() { + return name; + } + + public MongoClient createMongoClient(String connectionString) { + + CodecRegistry codecRegistry = fromRegistries( + fromCodecs(new UuidCodec(UuidRepresentation.STANDARD)), + MongoClientSettings.getDefaultCodecRegistry() + ); + + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(connectionString)) + .codecRegistry(codecRegistry) + .uuidRepresentation(UuidRepresentation.STANDARD) + .build(); + return MongoClients.create(settings); + } + + protected MongoDatabase getDatabase() { + return mongoDatabase; + } + + public MongoClient getClient() { + return this.client; + } +} diff --git a/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbDriverAdapter.java b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbDriverAdapter.java new file mode 100644 index 000000000..247ffe5a8 --- /dev/null +++ b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbDriverAdapter.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.mongodb.core; + +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; + +import java.util.function.Function; + +/** + * Special thanks to Justin Chu who authored the original NoSQLBench MongoDB ActivityType. + */ +@Service(value=DriverAdapter.class, selector ="mongodb") +public class MongodbDriverAdapter extends BaseDriverAdapter { + + @Override + public OpMapper getOpMapper() { + return new MongodbOpMapper(getSpaceCache()); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return s -> new MongoSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(MongoSpace.getConfigModel()); + } +} diff --git a/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbOpMapper.java b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbOpMapper.java new file mode 100644 index 000000000..3ab7d3722 --- /dev/null +++ b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongodbOpMapper.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.mongodb.core; + +import io.nosqlbench.engine.api.activityimpl.OpDispenser; +import io.nosqlbench.engine.api.activityimpl.OpMapper; +import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache; +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.engine.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class MongodbOpMapper implements OpMapper { + + private final DriverSpaceCache ctxcache; + + public MongodbOpMapper(DriverSpaceCache ctxcache) { + this.ctxcache = ctxcache; + } + + @Override + public OpDispenser apply(ParsedOp op) { + LongFunction ctxNamer = op.getAsFunctionOr("space","default"); + LongFunction ctxFunc = l -> ctxcache.get(ctxNamer.apply(l)); + return new MongoOpDispenser(ctxFunc, op); + } +} diff --git a/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOp.java b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOp.java new file mode 100644 index 000000000..7a7e2dff0 --- /dev/null +++ b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOp.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.mongodb.ops; + +import com.mongodb.client.MongoClient; +import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.CycleOp; +import org.bson.Document; +import org.bson.conversions.Bson; + +public class MongoOp implements CycleOp { + + private final MongoClient client; + private final Bson rqBson; + private final String database; + private int resultSize; + + // https://docs.mongodb.com/manual/reference/method/db.runCommand/#command-response + public MongoOp(MongoClient client, String database, Bson rqBson) { + this.client = client; + this.database = database; + this.rqBson = rqBson; + } + + @Override + public Document apply(long value) { + Document document = client.getDatabase(database).runCommand(rqBson); + int ok = document.getInteger("ok",0); + if (ok!=1) { + throw new MongoOpFailedException(rqBson, ok, document); + } + this.resultSize = document.getInteger("n"); + return document; + } + + @Override + public long getResultSize() { + return resultSize; + } +} diff --git a/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOpFailedException.java b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOpFailedException.java new file mode 100644 index 000000000..73274c990 --- /dev/null +++ b/adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/ops/MongoOpFailedException.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.mongodb.ops; + +import org.bson.Document; +import org.bson.conversions.Bson; + +public class MongoOpFailedException extends RuntimeException { + private final Bson rqBson; + private final int status; + private final Document document; + + public MongoOpFailedException(Bson rqBson, int status, Document document) { + this.rqBson = rqBson; + this.status = status; + this.document = document; + } + + @Override + public String getMessage() { + return "Error in MongoDB response, status=" + status + ", request=" + rqBson.toString() + " response=" + document.toString() + + ": " + super.getMessage(); + } +} diff --git a/driver-mongodb/src/main/resources/activities/mongodb-basic-uuid.yaml b/adapter-mongodb/src/main/resources/activities/mongodb-basic-uuid.yaml similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-basic-uuid.yaml rename to adapter-mongodb/src/main/resources/activities/mongodb-basic-uuid.yaml diff --git a/driver-mongodb/src/main/resources/activities/mongodb-basic.yaml b/adapter-mongodb/src/main/resources/activities/mongodb-basic.yaml similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-basic.yaml rename to adapter-mongodb/src/main/resources/activities/mongodb-basic.yaml diff --git a/driver-mongodb/src/main/resources/activities/mongodb-crud-basic.md b/adapter-mongodb/src/main/resources/activities/mongodb-crud-basic.md similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-crud-basic.md rename to adapter-mongodb/src/main/resources/activities/mongodb-crud-basic.md diff --git a/driver-mongodb/src/main/resources/activities/mongodb-crud-basic.yaml b/adapter-mongodb/src/main/resources/activities/mongodb-crud-basic.yaml similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-crud-basic.yaml rename to adapter-mongodb/src/main/resources/activities/mongodb-crud-basic.yaml diff --git a/driver-mongodb/src/main/resources/activities/mongodb-crud-dataset.md b/adapter-mongodb/src/main/resources/activities/mongodb-crud-dataset.md similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-crud-dataset.md rename to adapter-mongodb/src/main/resources/activities/mongodb-crud-dataset.md diff --git a/driver-mongodb/src/main/resources/activities/mongodb-crud-dataset.yaml b/adapter-mongodb/src/main/resources/activities/mongodb-crud-dataset.yaml similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-crud-dataset.yaml rename to adapter-mongodb/src/main/resources/activities/mongodb-crud-dataset.yaml diff --git a/driver-mongodb/src/main/resources/activities/mongodb-search-basic.md b/adapter-mongodb/src/main/resources/activities/mongodb-search-basic.md similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-search-basic.md rename to adapter-mongodb/src/main/resources/activities/mongodb-search-basic.md diff --git a/driver-mongodb/src/main/resources/activities/mongodb-search-basic.yaml b/adapter-mongodb/src/main/resources/activities/mongodb-search-basic.yaml similarity index 100% rename from driver-mongodb/src/main/resources/activities/mongodb-search-basic.yaml rename to adapter-mongodb/src/main/resources/activities/mongodb-search-basic.yaml diff --git a/driver-mongodb/src/main/resources/mongodb.md b/adapter-mongodb/src/main/resources/mongodb.md similarity index 100% rename from driver-mongodb/src/main/resources/mongodb.md rename to adapter-mongodb/src/main/resources/mongodb.md diff --git a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoAction.java b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoAction.java deleted file mode 100644 index a62dc8c4e..000000000 --- a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoAction.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.mongodb; - -import com.codahale.metrics.Timer; -import io.nosqlbench.engine.api.activityapi.core.SyncAction; -import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bson.Document; -import org.bson.conversions.Bson; - -import java.util.concurrent.TimeUnit; - -public class MongoAction implements SyncAction { - - private final static Logger logger = LogManager.getLogger(MongoAction.class); - - private final MongoActivity activity; - private final int slot; - - private OpSequence sequencer; - - public MongoAction(MongoActivity activity, int slot) { - this.activity = activity; - this.slot = slot; - } - - @Override - public void init() { - this.sequencer = activity.getOpSequencer(); - } - - @Override - public int runCycle(long cycle) { - ReadyMongoStatement rms; - Bson queryBson; - try (Timer.Context bindTime = activity.bindTimer.time()) { - rms = sequencer.apply(cycle); - queryBson = rms.bind(cycle); - - // Maybe show the query in log/console - only for diagnostic use - if (activity.isShowQuery()) { - logger.info("Query(cycle={}):\n{}", cycle, queryBson); - } - } - - long nanoStartTime = System.nanoTime(); - for (int i = 1; i <= activity.getMaxTries(); i++) { - activity.triesHisto.update(i); - - try (Timer.Context resultTime = activity.resultTimer.time()) { - // assuming the commands are one of these in the doc: - // https://docs.mongodb.com/manual/reference/command/nav-crud/ - Document resultDoc = activity.getDatabase().runCommand(queryBson, rms.getReadPreference()); - - long resultNanos = System.nanoTime() - nanoStartTime; - - // TODO: perhaps collect the operationTime from the resultDoc if any - // https://docs.mongodb.com/manual/reference/method/db.runCommand/#command-response - int ok = Double.valueOf((double) resultDoc.getOrDefault("ok", 0.0d)).intValue(); - if (ok == 1) { - // success - activity.resultSuccessTimer.update(resultNanos, TimeUnit.NANOSECONDS); - } - activity.resultSetSizeHisto.update(resultDoc.getInteger("n", 0)); - - return ok == 1 ? 0 : 1; - } catch (Exception e) { - logger.error("Failed to runCommand {} on cycle {}, tries {}", queryBson, cycle, i, e); - } - } - - throw new RuntimeException(String.format("Exhausted max tries (%s) on cycle %s", - activity.getMaxTries(), cycle)); - } -} diff --git a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java deleted file mode 100644 index 13a380069..000000000 --- a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivity.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.mongodb; - -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Timer; -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; -import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver; -import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner; -import io.nosqlbench.engine.api.activityapi.planning.SequencerType; -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; -import io.nosqlbench.engine.api.activityimpl.ActivityDef; -import io.nosqlbench.engine.api.activityimpl.SimpleActivity; -import io.nosqlbench.engine.api.metrics.ActivityMetrics; -import io.nosqlbench.engine.api.util.TagFilter; -import io.nosqlbench.virtdata.core.templates.ParsedTemplate; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bson.UuidRepresentation; -import org.bson.codecs.UuidCodec; -import org.bson.codecs.configuration.CodecRegistry; - -import java.util.List; -import java.util.Objects; -import java.util.function.Function; - -import static org.bson.codecs.configuration.CodecRegistries.fromCodecs; -import static org.bson.codecs.configuration.CodecRegistries.fromRegistries; - -public class MongoActivity extends SimpleActivity implements ActivityDefObserver { - - private final static Logger logger = LogManager.getLogger(MongoActivity.class); - - private String yamlLoc; - private String connectionString; - private String databaseName; - - private MongoClient client; - private MongoDatabase mongoDatabase; - private boolean showQuery; - - private OpSequence opSequence; - - Timer bindTimer; - Timer resultTimer; - Timer resultSuccessTimer; - Histogram triesHisto; - Histogram resultSetSizeHisto; - - public MongoActivity(ActivityDef activityDef) { - super(activityDef); - } - - @Override - public synchronized void onActivityDefUpdate(ActivityDef activityDef) { - super.onActivityDefUpdate(activityDef); - - // sanity check - yamlLoc = activityDef.getParams().getOptionalString("yaml", "workload") - .orElseThrow(() -> new IllegalArgumentException("yaml is not defined")); - connectionString = activityDef.getParams().getOptionalString("connection") - .orElseThrow(() -> new IllegalArgumentException("connection is not defined")); - // TODO: support multiple databases - databaseName = activityDef.getParams().getOptionalString("database") - .orElseThrow(() -> new IllegalArgumentException("database is not defined")); - } - - @Override - public void initActivity() { - logger.debug("initializing activity: " + this.activityDef.getAlias()); - onActivityDefUpdate(activityDef); - - opSequence = initOpSequencer(); - setDefaultsFromOpSequence(opSequence); - - client = createMongoClient(connectionString); - mongoDatabase = client.getDatabase(databaseName); - showQuery = activityDef.getParams().getOptionalBoolean("showquery") - .orElse(false); - bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits()); - resultTimer = ActivityMetrics.timer(activityDef, "result", this.getHdrDigits()); - resultSuccessTimer = ActivityMetrics.timer(activityDef, "result-success", this.getHdrDigits()); - resultSetSizeHisto = ActivityMetrics.histogram(activityDef, "resultset-size", this.getHdrDigits()); - triesHisto = ActivityMetrics.histogram(activityDef, "tries", this.getHdrDigits()); - } - - @Override - public void shutdownActivity() { - logger.debug("shutting down activity: " + this.activityDef.getAlias()); - if (client != null) { - client.close(); - } - } - - OpSequence initOpSequencer() { - SequencerType sequencerType = SequencerType.valueOf( - activityDef.getParams().getOptionalString("seq").orElse("bucket") - ); - SequencePlanner sequencer = new SequencePlanner<>(sequencerType); - - StmtsDocList stmtsDocList = StatementsLoader.loadPath( - logger, - yamlLoc, - activityDef.getParams(), - "activities" - ); - - String tagfilter = activityDef.getParams().getOptionalString("tags").orElse(""); - - TagFilter tagFilter = new TagFilter(tagfilter); - stmtsDocList.getStmts().stream().map(tagFilter::matchesTaggedResult).forEach(r -> logger.info(r.getLog())); - - List stmts = stmtsDocList.getStmts(tagfilter); - if (stmts.isEmpty()) { - logger.error("No statements found for this activity"); - } else { - for (OpTemplate stmt : stmts) { - ParsedTemplate parsed = stmt.getParsed().orElseThrow(); - String statement = parsed.getPositionalStatement(Function.identity()); - Objects.requireNonNull(statement); - - sequencer.addOp(new ReadyMongoStatement(stmt), stmt.getParamOrDefault("ratio", 1)); - } - } - - return sequencer.resolve(); - } - - MongoClient createMongoClient(String connectionString) { - CodecRegistry codecRegistry = fromRegistries(fromCodecs(new UuidCodec(UuidRepresentation.STANDARD)), - MongoClientSettings.getDefaultCodecRegistry()); - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(new ConnectionString(connectionString)) - .codecRegistry(codecRegistry) - .uuidRepresentation(UuidRepresentation.STANDARD) - .build(); - return MongoClients.create(settings); - } - - protected MongoDatabase getDatabase() { - return mongoDatabase; - } - - protected OpSequence getOpSequencer() { - return opSequence; - } - - protected boolean isShowQuery() { - return showQuery; - } - -} diff --git a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivityType.java b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivityType.java deleted file mode 100644 index fe43e667b..000000000 --- a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/MongoActivityType.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.mongodb; - -import io.nosqlbench.engine.api.activityapi.core.Action; -import io.nosqlbench.engine.api.activityapi.core.ActionDispenser; -import io.nosqlbench.engine.api.activityapi.core.ActivityType; -import io.nosqlbench.engine.api.activityimpl.ActivityDef; -import io.nosqlbench.nb.annotations.Service; - -@Service(value = ActivityType.class, selector = "mongodb") -public class MongoActivityType implements ActivityType { - - @Override - public MongoActivity getActivity(ActivityDef activityDef) { - return new MongoActivity(activityDef); - } - - @Override - public ActionDispenser getActionDispenser(MongoActivity activity) { - return new MongoActionDispenser(activity); - } - - private static class MongoActionDispenser implements ActionDispenser { - - private final MongoActivity activity; - - public MongoActionDispenser(MongoActivity activity) { - this.activity = activity; - } - - @Override - public Action getAction(int slot) { - return new MongoAction(activity, slot); - } - } -} diff --git a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java b/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java deleted file mode 100644 index dfad0ac79..000000000 --- a/driver-mongodb/src/main/java/io/nosqlbench/driver/mongodb/ReadyMongoStatement.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.mongodb; - -import com.mongodb.ReadPreference; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.virtdata.core.bindings.BindingsTemplate; -import io.nosqlbench.virtdata.core.templates.ParsedTemplate; -import io.nosqlbench.virtdata.core.templates.StringBindings; -import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate; -import org.bson.Document; -import org.bson.conversions.Bson; - -public class ReadyMongoStatement { - - private final StringBindings bindings; - private final ReadPreference readPreference; - - public ReadyMongoStatement(OpTemplate stmtDef) { - ParsedTemplate paramTemplate = new ParsedTemplate(stmtDef.getStmt().orElseThrow(), stmtDef.getBindings()); - BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints()); - StringBindingsTemplate template = new StringBindingsTemplate(stmtDef.getStmt().orElseThrow(), paramBindings); - - this.bindings = template.resolve(); - this.readPreference = stmtDef.getOptionalStringParam("readPreference") - .map(ReadPreference::valueOf) - .orElse(ReadPreference.primary()); - } - - public ReadPreference getReadPreference() { - return readPreference; - } - - public Bson bind(long value) { - return Document.parse(bindings.bind(value)); - } -} diff --git a/driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/MongoActivityTest.java b/driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/MongoActivityTest.java deleted file mode 100644 index 3ca78438a..000000000 --- a/driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/MongoActivityTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.mongodb; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import io.nosqlbench.engine.api.activityapi.planning.OpSequence; -import io.nosqlbench.engine.api.activityimpl.ActivityDef; - -import static org.assertj.core.api.Assertions.assertThat; - -public class MongoActivityTest { - - private ActivityDef activityDef; - - @BeforeEach - public void setup() { - String[] params = { - "yaml=activities/mongodb-basic.yaml", - "connection=mongodb://127.0.0.1", - "database=nosqlbench_testdb" - }; - activityDef = ActivityDef.parseActivityDef(String.join(";", params)); - } - - @Test - public void testInitOpSequencer() { - MongoActivity mongoActivity = new MongoActivity(activityDef); - mongoActivity.initActivity(); - - OpSequence sequence = mongoActivity.initOpSequencer(); - assertThat(sequence.getOps()).hasSize(3); - } -} diff --git a/driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java b/driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java deleted file mode 100644 index 7a6881eb5..000000000 --- a/driver-mongodb/src/test/java/io/nosqlbench/driver/mongodb/ReadyMongoStatementTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.driver.mongodb; - -import io.nosqlbench.engine.api.activityconfig.StatementsLoader; -import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate; -import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; -import io.nosqlbench.engine.api.activityimpl.ActivityDef; -import io.nosqlbench.virtdata.core.templates.BindPoint; -import io.nosqlbench.virtdata.core.templates.ParsedTemplate; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bson.conversions.Bson; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Objects; -import java.util.function.Function; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ReadyMongoStatementTest { - private final static Logger logger = LogManager.getLogger(ReadyMongoStatementTest.class); - - private ActivityDef activityDef; - private StmtsDocList stmtsDocList; - - @BeforeEach - public void setup() { - String[] params = { - "yaml=activities/mongodb-basic.yaml", - "database=nosqlbench_testdb", - }; - activityDef = ActivityDef.parseActivityDef(String.join(";", params)); - String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default"); - stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities"); - } - - @Test - public void testResolvePhaseRampup() { - String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("phase:rampup"); - - List stmts = stmtsDocList.getStmts(tagfilter); - assertThat(stmts).hasSize(1); - for (OpTemplate stmt : stmts) { - ParsedTemplate parsed = stmt.getParsed().orElseThrow(); - assertThat(parsed.getBindPoints()).hasSize(2); - - BindPoint seqKey = new BindPoint("seq_key", "Mod(1000000L); ToInt()"); - BindPoint seqValue = new BindPoint("seq_value", "Mod(1000000000L); Hash(); ToString() -> String"); - assertThat(parsed.getBindPoints()).containsExactly(seqKey, seqValue); - - String statement = parsed.getPositionalStatement(Function.identity()); - Objects.requireNonNull(statement); - - ReadyMongoStatement readyMongoStatement = new ReadyMongoStatement(stmt); - Bson bsonDoc = readyMongoStatement.bind(1L); - assertThat(bsonDoc).isNotNull(); - } - } - - @Test - public void testResolvePhaseMainRead() { - String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("phase:main,name:.*main-find"); - - List stmts = stmtsDocList.getStmts(tagfilter); - assertThat(stmts).hasSize(1); - for (OpTemplate stmt : stmts) { - ParsedTemplate parsed = stmt.getParsed().orElseThrow(); - assertThat(parsed.getBindPoints()).hasSize(1); - - BindPoint rwKey = new BindPoint("rw_key", "Uniform(0,1000000)->long; ToInt()"); - assertThat(parsed.getBindPoints()).containsExactly(rwKey); - - String statement = parsed.getPositionalStatement(Function.identity()); - Objects.requireNonNull(statement); - - ReadyMongoStatement readyMongoStatement = new ReadyMongoStatement(stmt); - Bson bsonDoc = readyMongoStatement.bind(1L); - assertThat(bsonDoc).isNotNull(); - } - } - - @Test - public void testResolvePhaseMainWrite() { - String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("phase:main,name:.*main-insert"); - - List stmts = stmtsDocList.getStmts(tagfilter); - assertThat(stmts).hasSize(1); - for (OpTemplate stmt : stmts) { - ParsedTemplate parsed = stmt.getParsed().orElseThrow(); - assertThat(parsed.getBindPoints()).hasSize(2); - - BindPoint rwKey = new BindPoint("rw_key", "Uniform(0,1000000)->long; ToInt()"); - BindPoint rwValue = new BindPoint("rw_value", "Uniform(0,1000000000)->int; Hash(); ToString() -> String"); - assertThat(parsed.getBindPoints()).containsExactly(rwKey, rwValue); - - String statement = parsed.getPositionalStatement(Function.identity()); - Objects.requireNonNull(statement); - - ReadyMongoStatement readyMongoStatement = new ReadyMongoStatement(stmt); - Bson bsonDoc = readyMongoStatement.bind(1L); - assertThat(bsonDoc).isNotNull(); - } - } -} diff --git a/nb5/pom.xml b/nb5/pom.xml index 2377296bd..912533be0 100644 --- a/nb5/pom.xml +++ b/nb5/pom.xml @@ -70,6 +70,12 @@ 4.17.15-SNAPSHOT + + io.nosqlbench + adapter-mongodb + 4.17.15-SNAPSHOT + + io.nosqlbench adapter-http diff --git a/pom.xml b/pom.xml index 74cedb3d9..f101ba2fe 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,7 @@ adapter-dynamodb adapter-diag adapter-stdout + adapter-mongodb