From 1894f5f148cb6fb0539f8c896dd619bc40681334 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 3 Oct 2024 14:08:49 -0400 Subject: [PATCH 1/7] Adding CreateDatabaseDDL option to the existing ops --- .../gcpspanner/GCPSpannerOpMapper.java | 2 + ...CPSpannerCreateDatabaseDdlOpDispenser.java | 74 +++++++++++++++++++ .../ops/GCPSpannerCreateDatabaseDdlOp.java | 71 ++++++++++++++++++ .../gcpspanner/types/GCPSpannerOpType.java | 1 + .../activities/create_table_ddl.yaml | 11 +++ .../src/main/resources/spanner.md | 6 +- 6 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index aa8533f9b..9b6ca8f3f 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -55,6 +55,8 @@ public class GCPSpannerOpMapper implements OpMapper> { logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); return switch (typeAndTarget.enumId) { + case create_database_ddl -> + new GCPSpannerCreateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction); case update_database_ddl -> new GCPSpannerUpdateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction); case insert -> diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java new file mode 100644 index 000000000..09f8ab314 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + + +import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerCreateDatabaseDdlOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerUpdateDatabaseDdlOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; +/** + * Dispenser class for creating databases of {@link GCPSpannerCreateDatabaseDdlOp}. + */ +public class GCPSpannerCreateDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerCreateDatabaseDdlOpDispenser.class); + private final LongFunction opFunction; + + /** + * Constructor for {@link GCPSpannerCreateDatabaseDdlOpDispenser}. + * + * @param adapter the {@link GCPSpannerDriverAdapter} instance + * @param op the {@link ParsedOp} instance + * @param targetFunction a {@link LongFunction} that provides the target string + */ + public GCPSpannerCreateDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + /** + * Creates a {@link LongFunction} that generates {@link GCPSpannerUpdateDatabaseDdlOp} instances. + * + * @param op the {@link ParsedOp} instance + * @return a {@link LongFunction} that generates {@link GCPSpannerUpdateDatabaseDdlOp} instances + */ + private LongFunction createOpFunction(ParsedOp op) { + return (l) -> new GCPSpannerCreateDatabaseDdlOp( + spaceFunction.apply(l).getSpanner(), + l, + targetFunction.apply(l), + spaceFunction.apply(l).getDbAdminClient(), + spaceFunction.apply(l).getInstanceId() + ); + } + + /** + * Retrieves an operation instance based on the provided value. + * + * @param value the long value used to generate the operation + * @return a {@link GCPSpannerBaseOp} instance + */ + @Override + public GCPSpannerBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java new file mode 100644 index 000000000..089fbb091 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner.ops; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.Spanner; +import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; + +/** + * This class represents an operation to create the database DDL (Data Definition Language) in Google Cloud Spanner. + * It extends the {@link GCPSpannerBaseOp} class and provides the implementation for applying the DDL update operation. + */ +public class GCPSpannerCreateDatabaseDdlOp extends GCPSpannerBaseOp { + private final String createDatbaseStatement; + private final DatabaseAdminClient dbAdminClient; + private final String instanceId; + + /** + * Constructs a new {@link GCPSpannerUpdateDatabaseDdlOp}. + * + * @param searchIndexClient the {@link Spanner} client + * @param requestParam the request parameter + * @param createDatbaseStatement the SQL statement to create the table + * @param dbAdminClient the {@link DatabaseAdminClient} to execute the DDL update + * @param instanceId the instance ID string representing the target spanner instance + */ + public GCPSpannerCreateDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String createDatbaseStatement, + DatabaseAdminClient dbAdminClient, String instanceId) { + super(searchIndexClient, requestParam); + this.createDatbaseStatement = createDatbaseStatement; + this.dbAdminClient = dbAdminClient; + this.instanceId = instanceId; + } + + /** + * Applies the DDL update operation. + * + * @param value the value to be used in the operation + * @return the result of the operation + * @throws RuntimeException if an error occurs during the operation + */ + @Override + public Object applyOp(long value) { + OperationFuture operation = dbAdminClient.createDatabase( + instanceId, + createDatbaseStatement, + null, + null); + try { + return operation.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java index e55551c8f..699a2537c 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java @@ -30,6 +30,7 @@ package io.nosqlbench.adapter.gcpspanner.types; * here */ public enum GCPSpannerOpType { + create_database_ddl, update_database_ddl, insert, execute_dml, diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml new file mode 100644 index 000000000..a74dfc05e --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml @@ -0,0 +1,11 @@ +scenarios: + default: + create_table_ddl: run driver=spanner tags==blocks:create_table_ddl service_account_file=TEMPLATE(service_account_file) + project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 + +blocks: + create_table_ddl: + ops: + op1: + create_database_ddl: | + CREATE DATABASE {database_id} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md b/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md index 70fc12fcc..ccc8cab82 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md @@ -19,12 +19,14 @@ to the adapter at runtime. ## Op Templates The Google Cloud Spanner adapter supports the following operations: + +* `create_database_ddl` - Data Definition Language operations such as creating and dropping databases. * `update_database_ddl` - Data Definition Language operations such as creating and dropping tables, indexes, etc. * `execute_dml` - Data Manipulation Language operations. Read only operations are supported at this time, including queries and vector queries. * `insert` - Insert a single record, vector or non-vector, of data into the database. + ## Examples - - +Checkout the sample workload files [here](./activities). --- From ccc15da0f0cf18ea72ff8a278c2c4634c7cc8fb6 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 3 Oct 2024 16:52:17 -0400 Subject: [PATCH 2/7] Adding Drop Database operation to Spanner adapter --- .../gcpspanner/GCPSpannerOpMapper.java | 2 + ...CPSpannerCreateDatabaseDdlOpDispenser.java | 4 + .../GCPSpannerDropDatabaseDdlOpDispenser.java | 77 +++++++++++++++++++ .../ops/GCPSpannerCreateDatabaseDdlOp.java | 2 +- .../ops/GCPSpannerDropDatabaseDdlOp.java | 70 +++++++++++++++++ .../gcpspanner/types/GCPSpannerOpType.java | 1 + ...able_ddl.yaml => create_database_ddl.yaml} | 4 +- .../activities/drop_database_ddl.yaml | 10 +++ .../activities/execute_index_ddl.yaml | 2 +- .../activities/execute_table_ddl.yaml | 2 +- 10 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java rename nb-adapters/adapter-gcp-spanner/src/main/resources/activities/{create_table_ddl.yaml => create_database_ddl.yaml} (62%) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index 9b6ca8f3f..d9a59fd2a 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -55,6 +55,8 @@ public class GCPSpannerOpMapper implements OpMapper> { logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); return switch (typeAndTarget.enumId) { + case drop_database_ddl -> + new GCPSpannerDropDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction); case create_database_ddl -> new GCPSpannerCreateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction); case update_database_ddl -> diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java index 09f8ab314..454baa4d4 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java @@ -26,8 +26,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; + /** * Dispenser class for creating databases of {@link GCPSpannerCreateDatabaseDdlOp}. + * + * @see + * CreateDatabaseRequest which can be a stretch goal to combine all of DB, Table(s), and Indexes into one-single call. */ public class GCPSpannerCreateDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser { private static final Logger logger = LogManager.getLogger(GCPSpannerCreateDatabaseDdlOpDispenser.class); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java new file mode 100644 index 000000000..e6edde42b --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + + +import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerDropDatabaseDdlOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +/** + * Dispenser class for creating databases of {@link GCPSpannerDropDatabaseDdlOp}. + * + * @see + * DropDatabaseRequest which can be a stretch goal to combine all of DB, Table(s), and Indexes into one-single call. + */ +public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerDropDatabaseDdlOpDispenser.class); + private final LongFunction opFunction; + + /** + * Constructor for {@link GCPSpannerDropDatabaseDdlOpDispenser}. + * + * @param adapter the {@link GCPSpannerDriverAdapter} instance + * @param op the {@link ParsedOp} instance + * @param targetFunction a {@link LongFunction} that provides the target string + */ + public GCPSpannerDropDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = DropOpFunction(op); + } + + /** + * Drops a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances. + * + * @param op the {@link ParsedOp} instance + * @return a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances + */ + private LongFunction DropOpFunction(ParsedOp op) { + return (l) -> new GCPSpannerDropDatabaseDdlOp( + spaceFunction.apply(l).getSpanner(), + l, + targetFunction.apply(l), + spaceFunction.apply(l).getDbAdminClient(), + spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString()) + ); + } + + /** + * Retrieves an operation instance based on the provided value. + * + * @param value the long value used to generate the operation + * @return a {@link GCPSpannerBaseOp} instance + */ + @Override + public GCPSpannerBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java index 089fbb091..90589c3ab 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java @@ -49,7 +49,7 @@ public class GCPSpannerCreateDatabaseDdlOp extends GCPSpannerBaseOp { } /** - * Applies the DDL update operation. + * Applies the DDL create operation. * * @param value the value to be used in the operation * @return the result of the operation diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java new file mode 100644 index 000000000..f5062c0cb --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner.ops; + +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.Spanner; + +/** + * This class represents an operation to Drop the database DDL (Data Definition Language) in Google Cloud Spanner. + * It extends the {@link GCPSpannerBaseOp} class and provides the implementation for applying the DDL update operation. + */ +public class GCPSpannerDropDatabaseDdlOp extends GCPSpannerBaseOp { + private final String databaseId; + private final DatabaseAdminClient dbAdminClient; + private final Database db; + + /** + * Constructs a new {@link GCPSpannerUpdateDatabaseDdlOp}. + * + * @param searchIndexClient the {@link Spanner} client + * @param requestParam the request parameter + * @param databaseId the database ID to be dropped + * @param dbAdminClient the {@link DatabaseAdminClient} to execute the DDL update + * @param db the {@link Database} to be dropped + */ + public GCPSpannerDropDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String databaseId, + DatabaseAdminClient dbAdminClient, Database db) { + super(searchIndexClient, requestParam); + this.databaseId = databaseId; + this.dbAdminClient = dbAdminClient; + this.db = db; + } + + /** + * Applies the DDL drop operation. + * + * @param value the value to be used in the operation + * @return the result of the operation + * @throws RuntimeException if an error occurs during the operation + */ + @Override + public Object applyOp(long value) { + try { + db.drop(); + } catch (Exception e) { + logger.warn("Error dropping database using the Database object: {}. Will re-try using the DBAdminClient now...", e.getMessage()); + try { + dbAdminClient.dropDatabase(databaseId, null); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + return null; + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java index 699a2537c..f26a06152 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java @@ -31,6 +31,7 @@ package io.nosqlbench.adapter.gcpspanner.types; */ public enum GCPSpannerOpType { create_database_ddl, + drop_database_ddl, update_database_ddl, insert, execute_dml, diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml similarity index 62% rename from nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml rename to nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml index a74dfc05e..64d419bf3 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_table_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml @@ -1,10 +1,10 @@ scenarios: default: - create_table_ddl: run driver=spanner tags==blocks:create_table_ddl service_account_file=TEMPLATE(service_account_file) + create_db_ddl: run driver=spanner tags==blocks:schema_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: - create_table_ddl: + schema_db: ops: op1: create_database_ddl: | diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml new file mode 100644 index 000000000..32ad67f91 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml @@ -0,0 +1,10 @@ +scenarios: + default: + create_db_ddl: run driver=spanner tags==blocks:schema_drop_db service_account_file=TEMPLATE(service_account_file) + project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 + +blocks: + schema_drop_db: + ops: + op1: + drop_database_ddl: "{database_id}" diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml index e7f67069d..47c712f08 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml @@ -9,5 +9,5 @@ blocks: ops: op1: update_database_ddl: | - CREATE VECTOR INDEX VectorsIndex ON vectors(value) + CREATE VECTOR INDEX IF NOT EXISTS VectorsIndex ON vectors(value) OPTIONS (distance_type = 'COSINE', tree_depth = 3, num_branches=1000, num_leaves = 1000000); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml index 5a33d38a4..be8dceb29 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml @@ -8,4 +8,4 @@ blocks: ops: op1: update_database_ddl: | - CREATE TABLE vectors (keycol STRING(100),value ARRAY(vector_length=>25) NOT NULL) PRIMARY KEY(keycol) + CREATE TABLE IF NOT EXISTS vectors (keycol STRING(100),value ARRAY(vector_length=>25) NOT NULL) PRIMARY KEY(keycol) From d65408555af0fcb6a4b5121ba7ab146443cc4850 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 3 Oct 2024 17:01:39 -0400 Subject: [PATCH 3/7] Fix all example workloads --- ...drant_batch_upsert_points_glove_25.run.xml | 14 ++++++++++ .run/spanner_schema_database_create.run.xml | 17 +++++++++++ mvn-defaults/pom.xml | 5 ++++ .../activities/create_database_ddl.yaml | 2 +- .../activities/drop_database_ddl.yaml | 2 +- .../activities/execute_index_ddl.yaml | 2 +- .../activities/execute_queryvector_dml.yaml | 2 +- .../activities/execute_table_ddl.yaml | 2 +- .../resources/activities/insert_vector.yaml | 2 +- nb-adapters/nb-adapters-included/pom.xml | 28 +++++++++---------- nb-adapters/pom.xml | 28 +++++++++---------- pom.xml | 7 +++++ 12 files changed, 77 insertions(+), 34 deletions(-) create mode 100644 .run/qdrant_batch_upsert_points_glove_25.run.xml create mode 100644 .run/spanner_schema_database_create.run.xml diff --git a/.run/qdrant_batch_upsert_points_glove_25.run.xml b/.run/qdrant_batch_upsert_points_glove_25.run.xml new file mode 100644 index 000000000..a4c967e89 --- /dev/null +++ b/.run/qdrant_batch_upsert_points_glove_25.run.xml @@ -0,0 +1,14 @@ + + + + + + \ No newline at end of file diff --git a/.run/spanner_schema_database_create.run.xml b/.run/spanner_schema_database_create.run.xml new file mode 100644 index 000000000..3dc7be332 --- /dev/null +++ b/.run/spanner_schema_database_create.run.xml @@ -0,0 +1,17 @@ + + + + + + \ No newline at end of file diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index bba4471a8..b3b3bc897 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -851,6 +851,11 @@ maven-deploy-plugin 3.0.0 + + org.codehaus.mojo + versions-maven-plugin + 2.17.1 + diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml index 64d419bf3..f06c8a22d 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - create_db_ddl: run driver=spanner tags==blocks:schema_db service_account_file=TEMPLATE(service_account_file) + create_db_ddl: run driver=spanner tags==block:schema_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml index 32ad67f91..75f153679 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - create_db_ddl: run driver=spanner tags==blocks:schema_drop_db service_account_file=TEMPLATE(service_account_file) + drop_db_ddl: run driver=spanner tags==block:schema_drop_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml index 47c712f08..ce0a691cd 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 # https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml index 9357e4b2f..b07cc6c18 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_dml: run driver=spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file) + execute_dml: run driver=spanner tags==block:execute_dml service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml index be8dceb29..fcb53fe2f 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml index d15c1bce3..21c77fea5 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml @@ -1,7 +1,7 @@ scenarios: default: insert_vector: >- - run driver=spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file) + run driver=spanner tags==block:insert_vector service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 7c854e88e..e812a1384 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -74,7 +74,7 @@ adapter-cqld4-include - true + false @@ -102,7 +102,7 @@ adapter-http-include - true + false @@ -130,7 +130,7 @@ adapter-tcp-include - true + false @@ -144,7 +144,7 @@ adapter-dataapi-include - true + false @@ -158,7 +158,7 @@ adapter-dynamodb-include - true + false @@ -172,7 +172,7 @@ adapter-mongodb-include - true + false @@ -186,7 +186,7 @@ adapter-pulsar-include - true + false @@ -200,7 +200,7 @@ adapter-s4j-include - true + false @@ -214,7 +214,7 @@ adapter-neo4j-include - true + false @@ -228,7 +228,7 @@ adapter-kafka-include - true + false @@ -242,7 +242,7 @@ adapter-amqp-include - true + false @@ -256,7 +256,7 @@ adapter-qdrant-include - true + false @@ -270,7 +270,7 @@ adapter-weaviate-include - true + false @@ -284,7 +284,7 @@ adapter-azure-aisearch-include - true + false diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index fc27812a8..d5dd20643 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -57,7 +57,7 @@ adapter-cqld4-module - true + false adapter-cqld4 @@ -77,7 +77,7 @@ adapter-http-module - true + false adapter-http @@ -97,7 +97,7 @@ adapter-tcp-module - true + false adapter-tcp @@ -107,7 +107,7 @@ adapter-dynamodb-module - true + false adapter-dynamodb @@ -117,7 +117,7 @@ adapter-mongodb-module - true + false adapter-mongodb @@ -127,7 +127,7 @@ adapter-neo4j-module - true + false adapter-neo4j @@ -137,7 +137,7 @@ adapter-pulsar-module - true + false adapter-pulsar @@ -147,7 +147,7 @@ adapter-s4j-module - true + false adapter-s4j @@ -157,7 +157,7 @@ adapter-kafka-module - true + false adapter-kafka @@ -167,7 +167,7 @@ adapter-amqp-module - true + false adapter-amqp @@ -177,7 +177,7 @@ adapter-dataapi-module - true + false adapter-dataapi @@ -187,7 +187,7 @@ adapter-qdrant-module - true + false adapter-qdrant @@ -197,7 +197,7 @@ adapter-weaviate-module - true + false adapter-weaviate @@ -207,7 +207,7 @@ adapter-azure-aisearch-module - true + false adapter-azure-aisearch diff --git a/pom.xml b/pom.xml index 117e5d587..ef356a0c9 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,13 @@ + + + + + org.codehaus.mojo + versions-maven-plugin + From 6bf10c63eb4f1d1dfc078b9dc5acbe1dffff4b81 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 3 Oct 2024 17:08:17 -0400 Subject: [PATCH 4/7] Fix up examples and add convenience run configs --- ...drant_batch_upsert_points_glove_25.run.xml | 14 ---------- .run/spanner_schema_database_create.run.xml | 17 ----------- mvn-defaults/pom.xml | 5 ---- .../activities/create_database_ddl.yaml | 2 +- .../activities/drop_database_ddl.yaml | 2 +- .../activities/execute_index_ddl.yaml | 2 +- .../activities/execute_queryvector_dml.yaml | 2 +- .../activities/execute_table_ddl.yaml | 2 +- .../resources/activities/insert_vector.yaml | 2 +- nb-adapters/nb-adapters-included/pom.xml | 28 +++++++++---------- nb-adapters/pom.xml | 28 +++++++++---------- pom.xml | 7 ----- 12 files changed, 34 insertions(+), 77 deletions(-) delete mode 100644 .run/qdrant_batch_upsert_points_glove_25.run.xml delete mode 100644 .run/spanner_schema_database_create.run.xml diff --git a/.run/qdrant_batch_upsert_points_glove_25.run.xml b/.run/qdrant_batch_upsert_points_glove_25.run.xml deleted file mode 100644 index a4c967e89..000000000 --- a/.run/qdrant_batch_upsert_points_glove_25.run.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.run/spanner_schema_database_create.run.xml b/.run/spanner_schema_database_create.run.xml deleted file mode 100644 index 3dc7be332..000000000 --- a/.run/spanner_schema_database_create.run.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index b3b3bc897..bba4471a8 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -851,11 +851,6 @@ maven-deploy-plugin 3.0.0 - - org.codehaus.mojo - versions-maven-plugin - 2.17.1 - diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml index f06c8a22d..64d419bf3 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - create_db_ddl: run driver=spanner tags==block:schema_db service_account_file=TEMPLATE(service_account_file) + create_db_ddl: run driver=spanner tags==blocks:schema_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml index 75f153679..32ad67f91 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - drop_db_ddl: run driver=spanner tags==block:schema_drop_db service_account_file=TEMPLATE(service_account_file) + create_db_ddl: run driver=spanner tags==blocks:schema_drop_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml index ce0a691cd..47c712f08 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 # https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml index b07cc6c18..9357e4b2f 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_dml: run driver=spanner tags==block:execute_dml service_account_file=TEMPLATE(service_account_file) + execute_dml: run driver=spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml index fcb53fe2f..be8dceb29 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml index 21c77fea5..d15c1bce3 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml @@ -1,7 +1,7 @@ scenarios: default: insert_vector: >- - run driver=spanner tags==block:insert_vector service_account_file=TEMPLATE(service_account_file) + run driver=spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index e812a1384..7c854e88e 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -74,7 +74,7 @@ adapter-cqld4-include - false + true @@ -102,7 +102,7 @@ adapter-http-include - false + true @@ -130,7 +130,7 @@ adapter-tcp-include - false + true @@ -144,7 +144,7 @@ adapter-dataapi-include - false + true @@ -158,7 +158,7 @@ adapter-dynamodb-include - false + true @@ -172,7 +172,7 @@ adapter-mongodb-include - false + true @@ -186,7 +186,7 @@ adapter-pulsar-include - false + true @@ -200,7 +200,7 @@ adapter-s4j-include - false + true @@ -214,7 +214,7 @@ adapter-neo4j-include - false + true @@ -228,7 +228,7 @@ adapter-kafka-include - false + true @@ -242,7 +242,7 @@ adapter-amqp-include - false + true @@ -256,7 +256,7 @@ adapter-qdrant-include - false + true @@ -270,7 +270,7 @@ adapter-weaviate-include - false + true @@ -284,7 +284,7 @@ adapter-azure-aisearch-include - false + true diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index d5dd20643..fc27812a8 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -57,7 +57,7 @@ adapter-cqld4-module - false + true adapter-cqld4 @@ -77,7 +77,7 @@ adapter-http-module - false + true adapter-http @@ -97,7 +97,7 @@ adapter-tcp-module - false + true adapter-tcp @@ -107,7 +107,7 @@ adapter-dynamodb-module - false + true adapter-dynamodb @@ -117,7 +117,7 @@ adapter-mongodb-module - false + true adapter-mongodb @@ -127,7 +127,7 @@ adapter-neo4j-module - false + true adapter-neo4j @@ -137,7 +137,7 @@ adapter-pulsar-module - false + true adapter-pulsar @@ -147,7 +147,7 @@ adapter-s4j-module - false + true adapter-s4j @@ -157,7 +157,7 @@ adapter-kafka-module - false + true adapter-kafka @@ -167,7 +167,7 @@ adapter-amqp-module - false + true adapter-amqp @@ -177,7 +177,7 @@ adapter-dataapi-module - false + true adapter-dataapi @@ -187,7 +187,7 @@ adapter-qdrant-module - false + true adapter-qdrant @@ -197,7 +197,7 @@ adapter-weaviate-module - false + true adapter-weaviate @@ -207,7 +207,7 @@ adapter-azure-aisearch-module - false + true adapter-azure-aisearch diff --git a/pom.xml b/pom.xml index ef356a0c9..117e5d587 100644 --- a/pom.xml +++ b/pom.xml @@ -99,13 +99,6 @@ - - - - - org.codehaus.mojo - versions-maven-plugin - From 903457a015c406ea94b698cee88349df8a7326df Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 3 Oct 2024 17:17:17 -0400 Subject: [PATCH 5/7] Fix up example workloads --- .../src/main/resources/activities/create_database_ddl.yaml | 2 +- .../src/main/resources/activities/drop_database_ddl.yaml | 2 +- .../src/main/resources/activities/execute_index_ddl.yaml | 2 +- .../src/main/resources/activities/execute_queryvector_dml.yaml | 2 +- .../src/main/resources/activities/execute_table_ddl.yaml | 2 +- .../src/main/resources/activities/insert_vector.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml index 64d419bf3..f06c8a22d 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - create_db_ddl: run driver=spanner tags==blocks:schema_db service_account_file=TEMPLATE(service_account_file) + create_db_ddl: run driver=spanner tags==block:schema_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml index 32ad67f91..75f153679 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - create_db_ddl: run driver=spanner tags==blocks:schema_drop_db service_account_file=TEMPLATE(service_account_file) + drop_db_ddl: run driver=spanner tags==block:schema_drop_db service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml index 47c712f08..ce0a691cd 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 # https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml index 9357e4b2f..b07cc6c18 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_dml: run driver=spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file) + execute_dml: run driver=spanner tags==block:execute_dml service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml index be8dceb29..fcb53fe2f 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml index d15c1bce3..21c77fea5 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml @@ -1,7 +1,7 @@ scenarios: default: insert_vector: >- - run driver=spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file) + run driver=spanner tags==block:insert_vector service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: From 9136b6f0e2ea77a2f5e697be0c870ebe0ed4f827 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 3 Oct 2024 18:05:05 -0400 Subject: [PATCH 6/7] additional tweaks --- .run/nb --list-drivers.run.xml | 4 ++-- .run/nb --list-scenarios.run.xml | 4 ++-- .run/spanner_schema_database_create.run.xml | 18 ++++++++++++++++++ .../activities/drop_database_ddl.yaml | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 .run/spanner_schema_database_create.run.xml diff --git a/.run/nb --list-drivers.run.xml b/.run/nb --list-drivers.run.xml index 55872229e..ab4f658c9 100644 --- a/.run/nb --list-drivers.run.xml +++ b/.run/nb --list-drivers.run.xml @@ -5,10 +5,10 @@