From 41fd1b283b8a64a8a8f04ec99693ca064bd5e712 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Thu, 12 Sep 2024 14:46:21 -0400 Subject: [PATCH 1/8] Initial skeleton for Spanner --- mvn-defaults/pom.xml | 8 +- nb-adapters/adapter-gcp-spanner/pom.xml | 55 ++++++++ .../gcpspanner/GCPSpannerAdapterUtils.java | 21 +++ .../gcpspanner/GCPSpannerDriverAdapter.java | 55 ++++++++ .../GCPSpannerDriverAdapterLoader.java | 32 +++++ .../gcpspanner/GCPSpannerOpMapper.java | 68 +++++++++ .../adapter/gcpspanner/GCPSpannerSpace.java | 132 ++++++++++++++++++ .../gcpspanner/ops/GCPSpannerBaseOp.java | 65 +++++++++ .../gcpspanner/types/GCPSpannerOpType.java | 24 ++++ .../src/main/resources/gcp-spanner.md | 67 +++++++++ nb-adapters/nb-adapters-included/pom.xml | 42 ++++-- nb-adapters/pom.xml | 38 +++-- 12 files changed, 578 insertions(+), 29 deletions(-) create mode 100644 nb-adapters/adapter-gcp-spanner/pom.xml create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapterLoader.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index c66371b88..e42d93c0e 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -415,6 +415,12 @@ 1.13.2 + + com.google.cloud + google-cloud-spanner + 6.74.0 + + @@ -438,7 +444,7 @@ org.apache.logging.log4j log4j-slf4j-impl - + org.apache.logging.log4j log4j-slf4j2-impl diff --git a/nb-adapters/adapter-gcp-spanner/pom.xml b/nb-adapters/adapter-gcp-spanner/pom.xml new file mode 100644 index 000000000..b4805eb84 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + adapter-gcp-spanner + jar + + + mvn-defaults + io.nosqlbench + ${revision} + ../../mvn-defaults + + + ${project.artifactId} + + A NoSQLBbench driver adapter module for the Google Spanner database. + + + + + io.nosqlbench + nb-annotations + ${revision} + compile + + + io.nosqlbench + adapters-api + ${revision} + compile + + + com.google.cloud + google-cloud-spanner + + + diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java new file mode 100644 index 000000000..1fc2bad27 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner; + +public class GCPSpannerAdapterUtils { + public static final String SPANNER = "gcp_spanner"; +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java new file mode 100644 index 000000000..97aef8007 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner; + +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.labels.NBLabels; + +import java.util.function.Function; + +import static io.nosqlbench.adapter.gcpspanner.GCPSpannerAdapterUtils.SPANNER; + +@Service(value = DriverAdapter.class, selector = SPANNER) +public class GCPSpannerDriverAdapter extends BaseDriverAdapter, GCPSpannerSpace> { + + public GCPSpannerDriverAdapter(NBComponent parentComponent, NBLabels labels) { + super(parentComponent, labels); + } + + @Override + public OpMapper> getOpMapper() { + return new GCPSpannerOpMapper(this); + } + + @Override + public Function getSpaceInitializer(NBConfiguration cfg) { + return (s) -> new GCPSpannerSpace(s, cfg); + } + + @Override + public NBConfigModel getConfigModel() { + return super.getConfigModel().add(GCPSpannerSpace.getConfigModel()); + } + +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapterLoader.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapterLoader.java new file mode 100644 index 000000000..73c0f42ea --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapterLoader.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner; + +import io.nosqlbench.adapter.diag.DriverAdapterLoader; +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.labels.NBLabels; + +import static io.nosqlbench.adapter.gcpspanner.GCPSpannerAdapterUtils.SPANNER; + +@Service(value = DriverAdapterLoader.class, selector = SPANNER) +public class GCPSpannerDriverAdapterLoader implements DriverAdapterLoader { + @Override + public GCPSpannerDriverAdapter load(NBComponent parent, NBLabels childLabels) { + return new GCPSpannerDriverAdapter(parent, childLabels); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java new file mode 100644 index 000000000..90b50560e --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner; + +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.types.GCPSpannerOpType; +import io.nosqlbench.adapters.api.activityimpl.OpDispenser; +import io.nosqlbench.adapters.api.activityimpl.OpMapper; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.engine.api.templating.TypeAndTarget; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class GCPSpannerOpMapper implements OpMapper> { + private static final Logger logger = LogManager.getLogger(GCPSpannerOpMapper.class); + private final GCPSpannerDriverAdapter adapter; + + /** + * Create a new {@code GCPSpannerOpMapper} implementing the {@link OpMapper}. + * interface. + * + * @param adapter The associated {@link GCPSpannerDriverAdapter} + */ + public GCPSpannerOpMapper(GCPSpannerDriverAdapter adapter) { + this.adapter = adapter; + } + + /** + * Given an instance of a {@link ParsedOp} returns the appropriate + * {@link GCPSpannerBaseOpDispenser} subclass. + * + * @param op The {@link ParsedOp} to be evaluated + * @return The correct {@link GCPSpannerBaseOpDispenser} subclass based on + * the op type + */ + @Override + public OpDispenser> apply(ParsedOp op) { + TypeAndTarget typeAndTarget = op.getTypeAndTarget(GCPSpannerOpType.class, + String.class, "type", "target"); + logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); + + return switch (typeAndTarget.enumId) { +// case delete_index -> new GCPSpannerDeleteIndexOpDispenser(adapter, op, typeAndTarget.targetFunction); + case create_table -> + new GCPSpannerCreateTableOpDispenser(adapter, op, typeAndTarget.targetFunction); +// case list_indexes -> new GCPSpannerListIndexesOpDispenser(adapter, op, typeAndTarget.targetFunction); +// case upload_documents -> new GCPSpannerUploadDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); +// case search_documents -> new GCPSpannerSearchDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); + +// default -> throw new RuntimeException( +// "Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + "mapping parsed op " + op); + }; + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java new file mode 100644 index 000000000..2950c42d4 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner; + +import com.google.cloud.spanner.*; +import io.nosqlbench.nb.api.config.standard.ConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfigModel; +import io.nosqlbench.nb.api.config.standard.NBConfiguration; +import io.nosqlbench.nb.api.config.standard.Param; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * @see ANN Docs + * @see Spanner Java Client + * @see Getting started in Java + * @see Authentication methods at Google + * @see Library Reference Doc + * @see DML Syntax + * @see + * @see + * @see + * @see + * @see + */ +public class GCPSpannerSpace implements AutoCloseable { + private final static Logger logger = LogManager.getLogger(GCPSpannerSpace.class); + private final String name; + private final NBConfiguration cfg; + protected Spanner spanner; + protected DatabaseAdminClient dbAdminClient; + protected DatabaseClient dbClient; + + /** + * Create a new {@code GCPSpannerSpace} Object which stores all stateful + * contextual information needed to interact with the Google Spanner + * database instance. + * + * @param name The name of this space + * @param cfg The configuration ({@link NBConfiguration}) for this nb run + */ + public GCPSpannerSpace(String name, NBConfiguration cfg) { + this.name = name; + this.cfg = cfg; + } + + public synchronized Spanner getSpanner() { + if (spanner == null) { + spanner = createSpanner(); + } + return spanner; + } + + public synchronized DatabaseAdminClient getDbAdminClient() { + return dbAdminClient; + } + + public synchronized DatabaseClient getDbClient() { + return dbClient; + } + + private Spanner createSpanner() { + if (/*cfg.getOptional("service_account_file").isEmpty() ||*/ + cfg.getOptional("database_id").isEmpty() || + cfg.getOptional("project_id").isEmpty() || + cfg.getOptional("instance_id").isEmpty()) { + throw new RuntimeException("You must provide all 'service_account_file', 'project_id', 'instance_id' & 'database_id' to configure a Google Spanner client"); + } + String projectId = cfg.get("project_id"); + String instanceId = cfg.get("instance_id"); + String databaseId = cfg.get("database_id"); + var spannerClient = SpannerOptions.newBuilder().setProjectId(projectId).build().getService(); + dbAdminClient = spannerClient.getDatabaseAdminClient(); + dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); +// var requiredToken = cfg.getOptional("token_file").map(Paths::get).map(tokenFilePath -> { +// try { +// return Files.readAllLines(tokenFilePath).getFirst(); +// } catch (IOException e) { +// String error = "Error while reading token from file:" + tokenFilePath; +// logger.error(error, e); +// throw new RuntimeException(e); +// } +// }).orElseGet(() -> cfg.getOptional("token").orElseThrow(() -> new RuntimeException( +// "You must provide either a 'token_file' or a 'token' to configure a Azure AI Search client"))); + +// logger.info(() -> "Creating new Azure AI Search Client with (masked) token/key [" +// + GCPSpannerAdapterUtils.maskDigits(requiredToken) + "], uri/endpoint [" + uri + "]"); +// +// var spannerBuilder = SpannerOptions().endpoint(uri); +// if (!requiredToken.isBlank()) { +// SpannerBuilder = SpannerBuilder.credential(new AzureKeyCredential(requiredToken)); +// } else { +// TokenCredential tokenCredential = new DefaultAzureCredentialBuilder().build(); +// SpannerBuilder = SpannerBuilder.credential(tokenCredential); +// } +// // Should we leave these below to leverage the SearchServiceVersion.getLatest()? +// String apiVersion = cfg.getOptional("api_version").orElse(SearchServiceVersion.V2024_07_01.name()); +// logger.warn( +// () -> "Latest search service version supported by this client is '" + SearchServiceVersion.getLatest() +// + "', but we're using '" + apiVersion + "' version. Ignore this warning if both are same."); + return spannerClient; + } + + public static NBConfigModel getConfigModel() { + return ConfigModel.of(GCPSpannerSpace.class) + .add(Param.optional("service_account_file", String.class, "the file to load the api token/key from. See https://cloud.google.com/docs/authentication/provide-credentials-adc#service-account")) +// .add(Param.defaultTo("token", "my-spanner-admin-key-changeme") +// .setDescription("the Spanner api token/key to use to connect to the database")) + .add(Param.optional("project_id", String.class,"Project ID containing the Spanner database. See https://cloud.google.com/resource-manager/docs/creating-managing-projects")) + .add(Param.optional("instance_id", String.class, "Spanner database's Instance ID containing. See https://cloud.google.com/spanner/docs/getting-started/java#create_an_instance")) + .add(Param.optional("database_id", String.class, "Spanner Database ID. See https://cloud.google.com/spanner/docs/getting-started/java#create_a_database")) + .asReadOnly(); + } + @Override + public void close() throws Exception { + spanner = null; + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java new file mode 100644 index 000000000..b04ee4051 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner.ops; + +import com.google.cloud.spanner.Spanner; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public abstract class GCPSpannerBaseOp implements CycleOp { + + protected final static Logger logger = LogManager.getLogger(GCPSpannerBaseOp.class); + + protected final Spanner spannerClient; + protected final T request; + protected final LongFunction apiCall; + + public GCPSpannerBaseOp(Spanner searchIndexClient, T requestParam) { + this.spannerClient = searchIndexClient; + this.request = requestParam; + this.apiCall = this::applyOp; + } + + public GCPSpannerBaseOp(Spanner spanner, T requestParam, LongFunction call) { + this.spannerClient = spanner; + this.request = requestParam; + this.apiCall = call; + } + + @Override + public final Object apply(long value) { + logger.trace(() -> "applying op: " + this); + + try { + Object result = applyOp(value); + + return result; + } catch (Exception rte) { + throw new RuntimeException(rte); + } + }; + + public abstract Object applyOp(long value); + + @Override + public String toString() { + return "GCPSpannerBaseOp(" + this.request.getClass().getSimpleName() + ")"; + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java new file mode 100644 index 000000000..32797ba7a --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2020-2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.gcpspanner.types; + +/** + * https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_statements + */ +public enum GCPSpannerOpType { + create_table, +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md b/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md new file mode 100644 index 000000000..ac0789357 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md @@ -0,0 +1,67 @@ +# Google Spanner driver adapter +The Azure AI Search driver adapter is a NoSQLBench adapter for the `azure-aisearch` driver, a Java driver +for connecting to and performing operations on an instance of a Azure AI Search vector database. The driver is +leveraged from GitHub at https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/search/azure-search-documents/. + +## Run Commands (Remove prior to merge) + +### Create Collection Schema +``` +java -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar weaviate_vector_live weaviate_vectors.rampup dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=Glove_25 weaviatehost=letsweave-czgwdrw9.weaviate.network token_file=${workspace_loc:/nosqlbench}/local/weaviate/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:weaviate_1255,instance:vectors,vendor:weaviate_wcd" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 +``` + +### Delete Collection +``` +java -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.delete_index dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=cosine azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azure_aisearch,instance:vectors,vendor:azure_aisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 +``` + +### List Indexes +``` +java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.list_indexes dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 +``` + +### Upload Documents +``` +java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.upload_documents dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 +``` + +### Search Documents +``` +java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.search_documents dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 +``` + + +## Activity Parameters + +The following parameters must be supplied to the adapter at runtime in order to successfully connect to an +instance of the [Azure AI Search database](https://learn.microsoft.com/en-us/rest/api/searchservice/?view=rest-searchservice-2024-07-01): + +* `token` - In order to use the Weaviate database you must have an account. Once the account is created you can [request + an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be + provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and + referenced via the `token_file` config option pointing to the path of the file. +* `endpoint` - When a collection/index is created in the database the URI (aka endpoint) must be specified as well. The adapter will + use the default value of `localhost:8080` if none is provided at runtime. +* `api_version` - the api version to be used by the search client. Defaults to the latest service/api version supported + by the version of client SDK. + +## Op Templates + +The Azure AI Search adapter supports [**all basic operations**](../java/io/nosqlbench/adapter/azure-aisearch/ops) supported by the [Java +client SDK published by Azure AI Search](https://github.com/weaviate/java-client). The official Azure AI Search API reference can be +found at https://learn.microsoft.com/en-us/rest/api/searchservice/operation-groups?view=rest-searchservice-2024-07-01. + +The operations include a full-fledged support for key APIs available in the Java SDK client. +The following are a couple high level API operations. + +* Create or Update Index +* Delete Index +* List Indexes +* Upload Documents (vectors) +* (Vector) Search Documents (vectors) + +## Examples + +Check out the [full example workload available here](./activities/azure_aisearch_vectors_live.yaml). + +--- diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index 7bd4e450a..e812a1384 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -74,7 +74,7 @@ adapter-cqld4-include - true + false @@ -102,7 +102,7 @@ adapter-http-include - true + false @@ -130,7 +130,7 @@ adapter-tcp-include - true + false @@ -144,7 +144,7 @@ adapter-dataapi-include - true + false @@ -158,7 +158,7 @@ adapter-dynamodb-include - true + false @@ -172,7 +172,7 @@ adapter-mongodb-include - true + false @@ -186,7 +186,7 @@ adapter-pulsar-include - true + false @@ -200,7 +200,7 @@ adapter-s4j-include - true + false @@ -214,7 +214,7 @@ adapter-neo4j-include - true + false @@ -228,7 +228,7 @@ adapter-kafka-include - true + false @@ -242,7 +242,7 @@ adapter-amqp-include - true + false @@ -256,7 +256,7 @@ adapter-qdrant-include - true + false @@ -270,7 +270,7 @@ adapter-weaviate-include - true + false @@ -284,7 +284,7 @@ adapter-azure-aisearch-include - true + false @@ -295,6 +295,20 @@ + + adapter-gcp-spanner-include + + true + + + + io.nosqlbench + adapter-gcp-spanner + ${revision} + + + + diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index e0c717aaa..d5dd20643 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -57,7 +57,7 @@ adapter-cqld4-module - true + false adapter-cqld4 @@ -77,7 +77,7 @@ adapter-http-module - true + false adapter-http @@ -97,7 +97,7 @@ adapter-tcp-module - true + false adapter-tcp @@ -107,7 +107,7 @@ adapter-dynamodb-module - true + false adapter-dynamodb @@ -117,7 +117,7 @@ adapter-mongodb-module - true + false adapter-mongodb @@ -127,7 +127,7 @@ adapter-neo4j-module - true + false adapter-neo4j @@ -137,7 +137,7 @@ adapter-pulsar-module - true + false adapter-pulsar @@ -147,7 +147,7 @@ adapter-s4j-module - true + false adapter-s4j @@ -157,7 +157,7 @@ adapter-kafka-module - true + false adapter-kafka @@ -167,7 +167,7 @@ adapter-amqp-module - true + false adapter-amqp @@ -177,7 +177,7 @@ adapter-dataapi-module - true + false adapter-dataapi @@ -187,7 +187,7 @@ adapter-qdrant-module - true + false adapter-qdrant @@ -197,7 +197,7 @@ adapter-weaviate-module - true + false adapter-weaviate @@ -207,12 +207,22 @@ adapter-azure-aisearch-module - true + false adapter-azure-aisearch + + adapter-gcp-spanner-module + + true + + + adapter-gcp-spanner + + + From b3144751e4e3539a103feb50cfd69079e9528efe Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Mon, 16 Sep 2024 17:24:34 -0400 Subject: [PATCH 2/8] compilation fix --- .../gcpspanner/GCPSpannerOpMapper.java | 1 + .../GCPSpannerBaseOpDispenser.java | 36 +++++++++++++++++++ .../GCPSpannerCreateTableOpDispenser.java | 29 +++++++++++++++ .../gcpspanner/ops/GCPSpannerBaseOp.java | 2 +- 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index 90b50560e..f9ba74db6 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -16,6 +16,7 @@ package io.nosqlbench.adapter.gcpspanner; +import io.nosqlbench.adapter.gcpspanner.opdispensers.*; import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; import io.nosqlbench.adapter.gcpspanner.types.GCPSpannerOpType; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java new file mode 100644 index 000000000..70ce58d63 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + +import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.GCPSpannerSpace; +import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +public class GCPSpannerBaseOpDispenser extends BaseOpDispenser, GCPSpannerSpace> { + + protected GCPSpannerBaseOpDispenser(DriverAdapter, ? extends GCPSpannerSpace> adapter, ParsedOp op) { + super(adapter, op); + } + + @Override + public GCPSpannerBaseOp getOp(long value) { + return null; + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java new file mode 100644 index 000000000..1febef2f7 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + +import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapters.api.templating.ParsedOp; + +import java.util.function.LongFunction; + +public class GCPSpannerCreateTableOpDispenser extends GCPSpannerBaseOpDispenser { + public GCPSpannerCreateTableOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java index b04ee4051..eb6b42b0e 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java @@ -54,7 +54,7 @@ public abstract class GCPSpannerBaseOp implements CycleOp { } catch (Exception rte) { throw new RuntimeException(rte); } - }; + } public abstract Object applyOp(long value); From 38970a6f23981986f0932565b6a07326f74eb393 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 17 Sep 2024 15:29:05 -0400 Subject: [PATCH 3/8] checkpoint --- .../gcpspanner/GCPSpannerOpMapper.java | 9 +-- .../adapter/gcpspanner/GCPSpannerSpace.java | 40 +++++-------- .../GCPSpannerBaseOpDispenser.java | 15 +++-- .../GCPSpannerCreateTableOpDispenser.java | 29 +++++++++- .../GCPSpannerInsertVectorOpDispenser.java | 57 +++++++++++++++++++ .../ops/GCPSpannerCreateTableOp.java | 50 ++++++++++++++++ .../ops/GCPSpannerInsertVectorOp.java | 41 +++++++++++++ .../gcpspanner/types/GCPSpannerOpType.java | 1 + 8 files changed, 203 insertions(+), 39 deletions(-) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index f9ba74db6..c2071da3b 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -55,15 +55,10 @@ public class GCPSpannerOpMapper implements OpMapper> { logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); return switch (typeAndTarget.enumId) { -// case delete_index -> new GCPSpannerDeleteIndexOpDispenser(adapter, op, typeAndTarget.targetFunction); case create_table -> new GCPSpannerCreateTableOpDispenser(adapter, op, typeAndTarget.targetFunction); -// case list_indexes -> new GCPSpannerListIndexesOpDispenser(adapter, op, typeAndTarget.targetFunction); -// case upload_documents -> new GCPSpannerUploadDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); -// case search_documents -> new GCPSpannerSearchDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction); - -// default -> throw new RuntimeException( -// "Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + "mapping parsed op " + op); + case insert_vector -> + new GCPSpannerInsertVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java index 2950c42d4..abdb6a3e4 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java @@ -73,6 +73,18 @@ public class GCPSpannerSpace implements AutoCloseable { return dbClient; } + public DatabaseId getDatabaseId() { + return DatabaseId.of(cfg.get("project_id"), cfg.get("instance_id"), cfg.get("database_id")); + } + + public String getInstanceId() { + return cfg.get("instance_id"); + } + + public String getDatabaseIdString() { + return cfg.get("database_id"); + } + private Spanner createSpanner() { if (/*cfg.getOptional("service_account_file").isEmpty() ||*/ cfg.getOptional("database_id").isEmpty() || @@ -86,32 +98,7 @@ public class GCPSpannerSpace implements AutoCloseable { var spannerClient = SpannerOptions.newBuilder().setProjectId(projectId).build().getService(); dbAdminClient = spannerClient.getDatabaseAdminClient(); dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); -// var requiredToken = cfg.getOptional("token_file").map(Paths::get).map(tokenFilePath -> { -// try { -// return Files.readAllLines(tokenFilePath).getFirst(); -// } catch (IOException e) { -// String error = "Error while reading token from file:" + tokenFilePath; -// logger.error(error, e); -// throw new RuntimeException(e); -// } -// }).orElseGet(() -> cfg.getOptional("token").orElseThrow(() -> new RuntimeException( -// "You must provide either a 'token_file' or a 'token' to configure a Azure AI Search client"))); -// logger.info(() -> "Creating new Azure AI Search Client with (masked) token/key [" -// + GCPSpannerAdapterUtils.maskDigits(requiredToken) + "], uri/endpoint [" + uri + "]"); -// -// var spannerBuilder = SpannerOptions().endpoint(uri); -// if (!requiredToken.isBlank()) { -// SpannerBuilder = SpannerBuilder.credential(new AzureKeyCredential(requiredToken)); -// } else { -// TokenCredential tokenCredential = new DefaultAzureCredentialBuilder().build(); -// SpannerBuilder = SpannerBuilder.credential(tokenCredential); -// } -// // Should we leave these below to leverage the SearchServiceVersion.getLatest()? -// String apiVersion = cfg.getOptional("api_version").orElse(SearchServiceVersion.V2024_07_01.name()); -// logger.warn( -// () -> "Latest search service version supported by this client is '" + SearchServiceVersion.getLatest() -// + "', but we're using '" + apiVersion + "' version. Ignore this warning if both are same."); return spannerClient; } @@ -127,6 +114,9 @@ public class GCPSpannerSpace implements AutoCloseable { } @Override public void close() throws Exception { + if (spanner != null) { + spanner.close(); + } spanner = null; } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java index 70ce58d63..0f9c37a59 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java @@ -23,14 +23,17 @@ import io.nosqlbench.adapter.gcpspanner.GCPSpannerSpace; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; -public class GCPSpannerBaseOpDispenser extends BaseOpDispenser, GCPSpannerSpace> { +import java.util.function.LongFunction; - protected GCPSpannerBaseOpDispenser(DriverAdapter, ? extends GCPSpannerSpace> adapter, ParsedOp op) { +public abstract class GCPSpannerBaseOpDispenser extends BaseOpDispenser, GCPSpannerSpace> { + protected final LongFunction targetFunction; + protected final LongFunction spaceFunction; + + protected GCPSpannerBaseOpDispenser(DriverAdapter, GCPSpannerSpace> adapter, ParsedOp op, + LongFunction targetFunction) { super(adapter, op); + this.targetFunction = targetFunction; + this.spaceFunction = adapter.getSpaceFunc(op); } - @Override - public GCPSpannerBaseOp getOp(long value) { - return null; - } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java index 1febef2f7..56cdc4692 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java @@ -17,13 +17,40 @@ package io.nosqlbench.adapter.gcpspanner.opdispensers; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapter.gcpspanner.GCPSpannerSpace; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerCreateTableOp; import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; public class GCPSpannerCreateTableOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerCreateTableOpDispenser.class); + private final LongFunction opFunction; + public GCPSpannerCreateTableOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op); + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + + return (l) -> new GCPSpannerCreateTableOp( + spaceFunction.apply(l).getSpanner(), + l, + op.getAsRequiredFunction("DDL", String.class).apply(l), + spaceFunction.apply(l).getDbAdminClient(), + spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString()) + ); + } + + @Override + public GCPSpannerBaseOp getOp(long value) { + return opFunction.apply(value); } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java new file mode 100644 index 000000000..aaa449991 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + +import com.google.cloud.spanner.Mutation; +import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerInsertVectorOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class GCPSpannerInsertVectorOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerInsertVectorOpDispenser.class); + private final LongFunction opFunction; + + public GCPSpannerInsertVectorOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + LongFunction vectorF= op.getAsRequiredFunction("vector", float[].class); + + return (l) -> new GCPSpannerInsertVectorOp( + spaceFunction.apply(l).getSpanner(), + l, + Mutation.newInsertBuilder(op.getStaticValue("table", java.lang.String.class)) + .set(op.getStaticValue("pkey", java.lang.String.class)).to(l) + .set("VectorData").toFloat32Array(vectorF.apply(l)) + .build(), + spaceFunction.apply(l).getDbClient() + ); + } + + @Override + public GCPSpannerBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java new file mode 100644 index 000000000..166bd754b --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.ops; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.spanner.*; +import com.google.common.collect.ImmutableList; +import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; + +public class GCPSpannerCreateTableOp extends GCPSpannerBaseOp { + private final String createTableStatement; + private final DatabaseAdminClient dbAdminClient; + private final Database db; + + public GCPSpannerCreateTableOp(Spanner searchIndexClient, Long requestParam, String createTableStatement, + DatabaseAdminClient dbAdminClient, Database db) { + super(searchIndexClient, requestParam); + this.createTableStatement = createTableStatement; + this.dbAdminClient = dbAdminClient; + this.db = db; + } + + @Override + public Object applyOp(long value) { + OperationFuture operation = dbAdminClient.updateDatabaseDdl( + db, + ImmutableList.of(createTableStatement), + null); + try { + return operation.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java new file mode 100644 index 000000000..666035b10 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.ops; + +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.DatabaseClient; +import com.google.cloud.spanner.Mutation; + +import java.util.Arrays; +import java.util.Collections; + +public class GCPSpannerInsertVectorOp extends GCPSpannerBaseOp { + private final Mutation mutation; + private final DatabaseClient dbClient; + + public GCPSpannerInsertVectorOp(Spanner searchIndexClient, Long requestParam, Mutation mutation, DatabaseClient dbClient) { + super(searchIndexClient, requestParam); + this.mutation = mutation; + this.dbClient = dbClient; + } + + @Override + public Object applyOp(long value) { + return dbClient.write(Collections.singletonList(mutation)); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java index 32797ba7a..e5ee95281 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java @@ -21,4 +21,5 @@ package io.nosqlbench.adapter.gcpspanner.types; */ public enum GCPSpannerOpType { create_table, + insert_vector, } From 0cb84c27c394ba328a0945c4d014f2b0c53fb345 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 20 Sep 2024 13:44:52 -0400 Subject: [PATCH 4/8] changes to allow for spanner --- nb-adapters/adapter-jdbc/pom.xml | 37 +++++++ .../adapter/jdbc/optypes/JDBCOp.java | 9 +- .../activities.baselinesv2/spanner.yaml | 98 +++++++++++++++++++ 3 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml diff --git a/nb-adapters/adapter-jdbc/pom.xml b/nb-adapters/adapter-jdbc/pom.xml index 987742f5e..23d3e9d28 100644 --- a/nb-adapters/adapter-jdbc/pom.xml +++ b/nb-adapters/adapter-jdbc/pom.xml @@ -33,6 +33,18 @@ into a PostegreSQL® compatible database leveraging HikariCP. + + + + + + + + + + + + @@ -68,6 +80,31 @@ + + + + + + + + + + + + + + com.google.cloud + google-cloud-spanner-jdbc + 2.22.0 + + + + com.google.cloud + google-cloud-spanner + 6.71.0 + + + diff --git a/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java b/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java index 11d06fe9c..fa9b6eb47 100644 --- a/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java +++ b/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java @@ -57,13 +57,14 @@ public abstract class JDBCOp implements CycleOp { else { String url = jdbcSpace.getConnConfig().getJdbcUrl(); Properties props = jdbcSpace.getConnConfig().getDataSourceProperties(); - props.put("user", jdbcSpace.getConnConfig().getUsername()); - props.put("password", jdbcSpace.getConnConfig().getPassword()); + if (jdbcSpace.getConnConfig().getUsername() != null) props.put("user", jdbcSpace.getConnConfig().getUsername()); + if (jdbcSpace.getConnConfig().getPassword() != null) props.put("password", jdbcSpace.getConnConfig().getPassword()); connection = DriverManager.getConnection(url, props); } - // Register 'vector' type - JDBCPgVector.addVectorType(connection); + if (connection.getMetaData().getDatabaseProductName().equals("PostgreSQL")) { + JDBCPgVector.addVectorType(connection); + } if (LOGGER.isDebugEnabled()) { LOGGER.debug("A new JDBC connection ({}) is successfully created: {}", diff --git a/nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml b/nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml new file mode 100644 index 000000000..9e44873d4 --- /dev/null +++ b/nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml @@ -0,0 +1,98 @@ +# run driver=jdbc workload="/path/to/postgresql-keyvalue.yaml" tags="block:schema" threads=AUTO cycles=4 url="jdbc:postgresql://host:port/database" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName=insectdb sslrootcert="/path/to/postgresql_certs/root.crt" -vv --show-stacktraces +min_version: "5.17.2" + +scenarios: + default: +# drop: run driver=jdbc tags==block:drop threads===1 cycles==UNDEF url="jdbc:postgresql://host:port/" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName="pgsql" sslrootcert="/path/to/postgresql_certs/root.crt" +# schema: run driver=jdbc tags==block:schema threads===1 cycles==UNDEF url="jdbc:postgresql://host:port/" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName="pgsql" sslrootcert="/path/to/postgresql_certs/root.crt" + train: run driver=jdbc tags==block:train threads=1 cycles===100 url="jdbc:cloudspanner:/projects/gcp-lcm-project/instances/stratperf-sep-24/databases/baselines" dml_batch=1 autoCommit=false databaseName="baselines" +# testann: run driver=jdbc tags==block:testann threads=AUTO cycles===TEMPLATE(main-cycles,1000) url="jdbc:postgresql://host:port/" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName="pgsql" sslrootcert="/path/to/postgresql_certs/root.crt" + +# "jdbc:cloudspanner:/projects/gcp-lcm-project/instances/stratperf-sep-24/databases/baselines?credentials=/home/cloudspanner-keys/my-key.json;autocommit=false"; + +bindings: + rw_key: ToString(); + train_floatlist: HdfFileToFloatList("/home/mwolters138/datasets/glove-25-angular.hdf5", "/train"); ToCqlVector(); + test_floatlist: HdfFileToFloatList("/home/mwolters138/datasets/glove-25-angular.hdf5", "/test"); ToCqlVector(); + relevant_indices: HdfFileToIntArray("/home/mwolters138/datasets/glove-25-angular.hdf5", "/neighbors") + +blocks: +# drop: +# ops: +# drop_vector_index: +# ddl: | +# DROP INDEX IF EXISTS idx_TEMPLATE(tablename,baseline)_TEMPLATE(indextype)_TEMPLATE(similarity_function); +# drop_table: +# ddl: | +# DROP TABLE IF EXISTS TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline); +# ## +# # NOTE: Do NOT enable this block for 'runall.sh' script +# # -------------------------------------------------- +# # drop_schema: +# # ddl: | +# # DROP SCHEMA IF EXISTS TEMPLATE(schemaname,public); +# +# schema: +# ops: +# create_schema: +# ddl: | +# CREATE SCHEMA IF NOT EXISTS TEMPLATE(schemaname,public); +# create_table: +# ddl: | +# CREATE TABLE IF NOT EXISTS TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) +# (key TEXT PRIMARY KEY, value vector(TEMPLATE(dimensions,5))); +# create_vector_index: +# ddl: | +# CREATE INDEX IF NOT EXISTS idx_TEMPLATE(tablename,baseline)_TEMPLATE(indextype)_TEMPLATE(similarity_function) +# ON TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) +# USING TEMPLATE(indextype) (value vector_TEMPLATE(similarity_function)_ops) +# WITH (TEMPLATE(indexopt)); + + train: + params: + prepared: true + ops: + main_insert: + dmlwrite: | + INSERT INTO TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) VALUES (?,?) + ON CONFLICT DO NOTHING; + prep_stmt_val_arr: | + {rw_key},{train_floatlist} + +# testann: +# params: +# prepared: true +# ops: +# # NOTE: right now this is only for cosine similarity. +# # in baselinetor, '<=>' is for cosine similarity +# # '<->' is for euclidean distance +# # '<#>' is for inner product +# main_select: +# dmlread: | +# SELECT * +# FROM TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) +# ORDER BY value <=> ? +# LIMIT TEMPLATE(top_k,100); +# prep_stmt_val_arr: | +# {test_floatlist} +# ################################# +# ## NOTE: +# # 1). The script blocks below are ONLY relevant with Vector relevancy score verification +# # 2). The "verifier-key" must match the Vector data identifier column name (e.g. primary key name) +# # right now the identifier must be a type that can be converted to int. +# verifier-key: "key" +# verifier-init: | +# relevancy=scriptingmetrics.newRelevancyMeasures(_parsed_op); +# k=TEMPLATE(top_k,100) +# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("recall",k)); +# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("precision",k)); +# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("F1",k)); +# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("RR",k)); +# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("AP",k)); +# verifier: | +# // driver-specific function +# actual_indices=pgvec_utils.getValueListForVerifierKey(result); +# // driver-agnostic function +# relevancy.accept({relevant_indices},actual_indices); +# // because we are "verifying" although this needs to be reorganized +# return true; From 9d5824b6506880788114f32c11c103288ab6fb58 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 24 Sep 2024 16:16:09 -0400 Subject: [PATCH 5/8] add ops for dml and ddl support --- .../gcpspanner/GCPSpannerAdapterUtils.java | 6 ++ .../gcpspanner/GCPSpannerOpMapper.java | 6 +- .../adapter/gcpspanner/GCPSpannerSpace.java | 2 +- .../GCPSpannerExecuteDmlOpDispenser.java | 57 +++++++++++++++++++ ...PSpannerUpdateDatabaseDdlOpDispenser.java} | 17 +++--- .../gcpspanner/ops/GCPSpannerBaseOp.java | 6 -- .../ops/GCPSpannerExecuteDmlOp.java | 39 +++++++++++++ .../ops/GCPSpannerInsertVectorOp.java | 1 - ...ava => GCPSpannerUpdateDatabaseDdlOp.java} | 6 +- .../gcpspanner/types/GCPSpannerOpType.java | 15 ++++- 10 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java rename nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/{GCPSpannerCreateTableOpDispenser.java => GCPSpannerUpdateDatabaseDdlOpDispenser.java} (70%) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java rename nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/{GCPSpannerCreateTableOp.java => GCPSpannerUpdateDatabaseDdlOp.java} (84%) diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java index 1fc2bad27..55cf996d4 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java @@ -16,6 +16,12 @@ package io.nosqlbench.adapter.gcpspanner; +import com.google.cloud.spanner.ResultSet; + public class GCPSpannerAdapterUtils { public static final String SPANNER = "gcp_spanner"; + + public static int[] getKeyArrayFromResultSet(ResultSet rs) { + return rs.getLongList(0).stream().mapToInt(Math::toIntExact).toArray(); + } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index c2071da3b..f066ee4cc 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -55,10 +55,12 @@ public class GCPSpannerOpMapper implements OpMapper> { logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'"); return switch (typeAndTarget.enumId) { - case create_table -> - new GCPSpannerCreateTableOpDispenser(adapter, op, typeAndTarget.targetFunction); + case update_database_ddl -> + new GCPSpannerUpdateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction); case insert_vector -> new GCPSpannerInsertVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); + case execute_dml -> + new GCPSpannerExecuteDmlOpDispenser(adapter, op, typeAndTarget.targetFunction); }; } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java index abdb6a3e4..13f7feabf 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java @@ -113,7 +113,7 @@ public class GCPSpannerSpace implements AutoCloseable { .asReadOnly(); } @Override - public void close() throws Exception { + public void close() { if (spanner != null) { spanner.close(); } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java new file mode 100644 index 000000000..7e567da70 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + +import com.google.cloud.spanner.Statement; +import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerExecuteDmlOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.function.LongFunction; + +public class GCPSpannerExecuteDmlOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerExecuteDmlOpDispenser.class); + private final LongFunction opFunction; + + public GCPSpannerExecuteDmlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.opFunction = createOpFunction(op); + } + + private LongFunction createOpFunction(ParsedOp op) { + + return (l) -> new GCPSpannerExecuteDmlOp( + spaceFunction.apply(l).getSpanner(), + l, + generateStatement(op.getAsRequiredFunction("DML", String.class).apply(l)), + spaceFunction.apply(l).getDbClient() + ); + } + + private Statement generateStatement(String dml) { + return Statement.of(dml); + } + + @Override + public GCPSpannerBaseOp getOp(long value) { + return opFunction.apply(value); + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java similarity index 70% rename from nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java rename to nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java index 56cdc4692..c3224db1d 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateTableOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java @@ -17,30 +17,27 @@ package io.nosqlbench.adapter.gcpspanner.opdispensers; -import com.google.cloud.spanner.Database; -import com.google.cloud.spanner.DatabaseAdminClient; import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; -import io.nosqlbench.adapter.gcpspanner.GCPSpannerSpace; import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; -import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerCreateTableOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerUpdateDatabaseDdlOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; -public class GCPSpannerCreateTableOpDispenser extends GCPSpannerBaseOpDispenser { - private static final Logger logger = LogManager.getLogger(GCPSpannerCreateTableOpDispenser.class); - private final LongFunction opFunction; +public class GCPSpannerUpdateDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerUpdateDatabaseDdlOpDispenser.class); + private final LongFunction opFunction; - public GCPSpannerCreateTableOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + public GCPSpannerUpdateDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } - private LongFunction createOpFunction(ParsedOp op) { + private LongFunction createOpFunction(ParsedOp op) { - return (l) -> new GCPSpannerCreateTableOp( + return (l) -> new GCPSpannerUpdateDatabaseDdlOp( spaceFunction.apply(l).getSpanner(), l, op.getAsRequiredFunction("DDL", String.class).apply(l), diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java index eb6b42b0e..4565630b9 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java @@ -37,12 +37,6 @@ public abstract class GCPSpannerBaseOp implements CycleOp { this.apiCall = this::applyOp; } - public GCPSpannerBaseOp(Spanner spanner, T requestParam, LongFunction call) { - this.spannerClient = spanner; - this.request = requestParam; - this.apiCall = call; - } - @Override public final Object apply(long value) { logger.trace(() -> "applying op: " + this); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java new file mode 100644 index 000000000..992bc95d7 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.ops; + +import com.google.cloud.spanner.*; + +public class GCPSpannerExecuteDmlOp extends GCPSpannerBaseOp { + private final Statement statement; + private final DatabaseClient dbClient; + + public GCPSpannerExecuteDmlOp(Spanner spanner, Long requestParam, Statement statement, + DatabaseClient dbClient) { + super(spanner, requestParam); + this.statement = statement; + this.dbClient = dbClient; + } + + @Override + public Object applyOp(long value) { + try (ReadContext context = dbClient.singleUse()) { + return context.executeQuery(statement); + } + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java index 666035b10..a39c58454 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java @@ -21,7 +21,6 @@ import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.Mutation; -import java.util.Arrays; import java.util.Collections; public class GCPSpannerInsertVectorOp extends GCPSpannerBaseOp { diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java similarity index 84% rename from nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java rename to nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java index 166bd754b..a0903bc6b 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateTableOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java @@ -22,13 +22,13 @@ import com.google.cloud.spanner.*; import com.google.common.collect.ImmutableList; import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; -public class GCPSpannerCreateTableOp extends GCPSpannerBaseOp { +public class GCPSpannerUpdateDatabaseDdlOp extends GCPSpannerBaseOp { private final String createTableStatement; private final DatabaseAdminClient dbAdminClient; private final Database db; - public GCPSpannerCreateTableOp(Spanner searchIndexClient, Long requestParam, String createTableStatement, - DatabaseAdminClient dbAdminClient, Database db) { + public GCPSpannerUpdateDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String createTableStatement, + DatabaseAdminClient dbAdminClient, Database db) { super(searchIndexClient, requestParam); this.createTableStatement = createTableStatement; this.dbAdminClient = dbAdminClient; diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java index e5ee95281..67477d341 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java @@ -17,9 +17,20 @@ package io.nosqlbench.adapter.gcpspanner.types; /** - * https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_statements + * All the spanner rpc api calls are defined here, representing a + * guide to the set of operations we should define if we want to implement full Spanner api support. + *

+ * NOTE that the vector search functionality is still in pre-GA and is not available through rpc calls other than simply + * calling ExecuteSql. The SQL functionality related to vector indices is documented + * here + *

+ * KNN and ANN search through Google SQL are documented respectively + * here + * and + * here */ public enum GCPSpannerOpType { - create_table, + update_database_ddl, insert_vector, + execute_dml, } From eb33818ce0969c5b2e8716f4f71961466ccc667b Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Wed, 25 Sep 2024 16:26:44 -0400 Subject: [PATCH 6/8] Initial implementation of gcp spanner adapter reverting jdbc changes for pr renames and cleanup --- .../gcpspanner/GCPSpannerAdapterUtils.java | 9 +- .../gcpspanner/GCPSpannerOpMapper.java | 4 +- .../adapter/gcpspanner/GCPSpannerSpace.java | 18 ++-- .../GCPSpannerBaseOpDispenser.java | 19 ++++ .../GCPSpannerExecuteDmlOpDispenser.java | 33 ++++++- .../GCPSpannerInsertOpDispenser.java | 99 +++++++++++++++++++ .../GCPSpannerInsertVectorOpDispenser.java | 57 ----------- ...CPSpannerUpdateDatabaseDdlOpDispenser.java | 25 ++++- .../gcpspanner/ops/GCPSpannerBaseOp.java | 35 ++++++- .../ops/GCPSpannerExecuteDmlOp.java | 18 ++++ ...tVectorOp.java => GCPSpannerInsertOp.java} | 22 ++++- .../ops/GCPSpannerUpdateDatabaseDdlOp.java | 20 ++++ .../gcpspanner/types/GCPSpannerOpType.java | 2 +- .../activities/execute_index_ddl.yaml | 13 +++ .../activities/execute_queryvector_dml.yaml | 18 ++++ .../activities/execute_table_ddl.yaml | 11 +++ .../resources/activities/insert_vector.yaml | 18 ++++ .../src/main/resources/gcp-spanner.md | 71 ++++--------- nb-adapters/adapter-jdbc/pom.xml | 44 --------- .../adapter/jdbc/optypes/JDBCOp.java | 9 +- .../activities.baselinesv2/spanner.yaml | 98 ------------------ nb-adapters/nb-adapters-included/pom.xml | 28 +++--- nb-adapters/pom.xml | 28 +++--- 23 files changed, 389 insertions(+), 310 deletions(-) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java delete mode 100644 nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java rename nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/{GCPSpannerInsertVectorOp.java => GCPSpannerInsertOp.java} (54%) create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml create mode 100644 nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml delete mode 100644 nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java index 55cf996d4..2b65059ab 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java @@ -18,10 +18,17 @@ package io.nosqlbench.adapter.gcpspanner; import com.google.cloud.spanner.ResultSet; +import java.util.ArrayList; +import java.util.List; + public class GCPSpannerAdapterUtils { public static final String SPANNER = "gcp_spanner"; public static int[] getKeyArrayFromResultSet(ResultSet rs) { - return rs.getLongList(0).stream().mapToInt(Math::toIntExact).toArray(); + List values = new ArrayList<>(); + while(rs.next()) { + values.add(Integer.valueOf(rs.getString(0))); + } + return values.stream().mapToInt(i -> i).toArray(); } } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java index f066ee4cc..aa8533f9b 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java @@ -57,8 +57,8 @@ public class GCPSpannerOpMapper implements OpMapper> { return switch (typeAndTarget.enumId) { case update_database_ddl -> new GCPSpannerUpdateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction); - case insert_vector -> - new GCPSpannerInsertVectorOpDispenser(adapter, op, typeAndTarget.targetFunction); + case insert -> + new GCPSpannerInsertOpDispenser(adapter, op, typeAndTarget.targetFunction); case execute_dml -> new GCPSpannerExecuteDmlOpDispenser(adapter, op, typeAndTarget.targetFunction); }; diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java index 13f7feabf..09d0d92b9 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java @@ -31,8 +31,8 @@ import org.apache.logging.log4j.Logger; * @see Authentication methods at Google * @see Library Reference Doc * @see DML Syntax - * @see - * @see + * @see spanner rpc api calls + * @see SQL functionality related to vector indices * @see * @see * @see @@ -60,7 +60,7 @@ public class GCPSpannerSpace implements AutoCloseable { public synchronized Spanner getSpanner() { if (spanner == null) { - spanner = createSpanner(); + createSpanner(); } return spanner; } @@ -85,8 +85,8 @@ public class GCPSpannerSpace implements AutoCloseable { return cfg.get("database_id"); } - private Spanner createSpanner() { - if (/*cfg.getOptional("service_account_file").isEmpty() ||*/ + private void createSpanner() { + if ( cfg.getOptional("database_id").isEmpty() || cfg.getOptional("project_id").isEmpty() || cfg.getOptional("instance_id").isEmpty()) { @@ -95,18 +95,14 @@ public class GCPSpannerSpace implements AutoCloseable { String projectId = cfg.get("project_id"); String instanceId = cfg.get("instance_id"); String databaseId = cfg.get("database_id"); - var spannerClient = SpannerOptions.newBuilder().setProjectId(projectId).build().getService(); - dbAdminClient = spannerClient.getDatabaseAdminClient(); + spanner = SpannerOptions.newBuilder().setProjectId(projectId).build().getService(); + dbAdminClient = spanner.getDatabaseAdminClient(); dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); - - return spannerClient; } public static NBConfigModel getConfigModel() { return ConfigModel.of(GCPSpannerSpace.class) .add(Param.optional("service_account_file", String.class, "the file to load the api token/key from. See https://cloud.google.com/docs/authentication/provide-credentials-adc#service-account")) -// .add(Param.defaultTo("token", "my-spanner-admin-key-changeme") -// .setDescription("the Spanner api token/key to use to connect to the database")) .add(Param.optional("project_id", String.class,"Project ID containing the Spanner database. See https://cloud.google.com/resource-manager/docs/creating-managing-projects")) .add(Param.optional("instance_id", String.class, "Spanner database's Instance ID containing. See https://cloud.google.com/spanner/docs/getting-started/java#create_an_instance")) .add(Param.optional("database_id", String.class, "Spanner Database ID. See https://cloud.google.com/spanner/docs/getting-started/java#create_a_database")) diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java index 0f9c37a59..e7550a3a0 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java @@ -25,10 +25,29 @@ import io.nosqlbench.adapters.api.templating.ParsedOp; import java.util.function.LongFunction; +/** + * Abstract base class for GCP Spanner operation dispensers. + * This class extends the BaseOpDispenser and provides common functionality + * for creating GCP Spanner operations. + */ public abstract class GCPSpannerBaseOpDispenser extends BaseOpDispenser, GCPSpannerSpace> { + /** + * A function that provides the target string based on a long input. + */ protected final LongFunction targetFunction; + + /** + * A function that provides the GCP Spanner space based on a long input. + */ protected final LongFunction spaceFunction; + /** + * Constructs a new GCPSpannerBaseOpDispenser. + * + * @param adapter the driver adapter for GCP Spanner operations + * @param op the parsed operation + * @param targetFunction a function that provides the target string + */ protected GCPSpannerBaseOpDispenser(DriverAdapter, GCPSpannerSpace> adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java index 7e567da70..9c2fd0d83 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java @@ -27,29 +27,58 @@ import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; +/** + * Dispenser class for creating GCP Spanner Execute DML operations. + * This class extends the GCPSpannerBaseOpDispenser and provides functionality + * to create and configure GCPSpannerExecuteDmlOp instances. + */ public class GCPSpannerExecuteDmlOpDispenser extends GCPSpannerBaseOpDispenser { private static final Logger logger = LogManager.getLogger(GCPSpannerExecuteDmlOpDispenser.class); private final LongFunction opFunction; + /** + * Constructs a new GCPSpannerExecuteDmlOpDispenser. + * + * @param adapter the driver adapter for GCP Spanner operations + * @param op the parsed operation + * @param targetFunction a function that provides the target string + */ public GCPSpannerExecuteDmlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } + /** + * Creates a function that generates GCPSpannerExecuteDmlOp instances. + * + * @param op the parsed operation + * @return a function that generates GCPSpannerExecuteDmlOp instances + */ private LongFunction createOpFunction(ParsedOp op) { - return (l) -> new GCPSpannerExecuteDmlOp( spaceFunction.apply(l).getSpanner(), l, - generateStatement(op.getAsRequiredFunction("DML", String.class).apply(l)), + generateStatement(targetFunction.apply(l)), spaceFunction.apply(l).getDbClient() ); } + /** + * Generates a Spanner Statement from a DML string. + * + * @param dml the DML string + * @return the generated Statement + */ private Statement generateStatement(String dml) { return Statement.of(dml); } + /** + * Retrieves the GCP Spanner operation for the given value. + * + * @param value the input value + * @return the GCP Spanner operation + */ @Override public GCPSpannerBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java new file mode 100644 index 000000000..299a0a642 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.nosqlbench.adapter.gcpspanner.opdispensers; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Value; +import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; +import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerInsertOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collections; +import java.util.Map; +import java.util.function.LongFunction; + +/** + * This class is responsible for dispensing GCP Spanner insert vector operations. + * It extends the GCPSpannerBaseOpDispenser and provides the necessary implementation + * to create and configure GCPSpannerInsertVectorOp instances. + */ +public class GCPSpannerInsertOpDispenser extends GCPSpannerBaseOpDispenser { + private static final Logger logger = LogManager.getLogger(GCPSpannerInsertOpDispenser.class); + private final LongFunction queryParamsFunction; + + /** + * Constructs a new GCPSpannerInsertVectorOpDispenser. + * + * @param adapter the GCP Spanner driver adapter + * @param op the parsed operation + * @param targetFunction a function that provides the target table name based on a long value + */ + public GCPSpannerInsertOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { + super(adapter, op, targetFunction); + this.queryParamsFunction = createParamsFunction(op); + } + + /** + * Creates a function that provides query parameters based on a long value. + * + * @param op the parsed operation + * @return a function that provides query parameters + */ + private LongFunction createParamsFunction(ParsedOp op) { + return op.getAsOptionalFunction("query_params", Map.class) + .orElse(_ -> Collections.emptyMap()); + } + + /** + * Returns a GCPSpannerInsertVectorOp instance configured with the provided value. + * + * @param value the value used to configure the operation + * @return a configured GCPSpannerInsertVectorOp instance + */ + @Override + public GCPSpannerBaseOp getOp(long value) { + Mutation.WriteBuilder builder = Mutation.newInsertBuilder(targetFunction.apply(value)); + Map params = queryParamsFunction.apply(value); + for (Map.Entry entry : params.entrySet()) { + builder.set(entry.getKey()).to(convertToValue(entry)); + } + return new GCPSpannerInsertOp( + spaceFunction.apply(value).getSpanner(), + value, + builder.build(), + spaceFunction.apply(value).getDbClient() + ); + } + + private Value convertToValue(Map.Entry entry) { + return switch(entry.getValue()) { + case String s -> Value.string(s); + case Integer i -> Value.int64(i); + case Long l -> Value.int64(l); + case Double d -> Value.float64(d); + case Float f -> Value.float32(f); + case long[] larr -> Value.int64Array(larr); + case float[] farr -> Value.float32Array(farr); + case double[] darr -> Value.float64Array(darr); + default -> throw new IllegalArgumentException("Unsupported value type: " + entry.getValue().getClass()); + }; + } +} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java deleted file mode 100644 index aaa449991..000000000 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertVectorOpDispenser.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2024 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.nosqlbench.adapter.gcpspanner.opdispensers; - -import com.google.cloud.spanner.Mutation; -import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter; -import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp; -import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerInsertVectorOp; -import io.nosqlbench.adapters.api.templating.ParsedOp; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.function.LongFunction; - -public class GCPSpannerInsertVectorOpDispenser extends GCPSpannerBaseOpDispenser { - private static final Logger logger = LogManager.getLogger(GCPSpannerInsertVectorOpDispenser.class); - private final LongFunction opFunction; - - public GCPSpannerInsertVectorOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { - super(adapter, op, targetFunction); - this.opFunction = createOpFunction(op); - } - - private LongFunction createOpFunction(ParsedOp op) { - LongFunction vectorF= op.getAsRequiredFunction("vector", float[].class); - - return (l) -> new GCPSpannerInsertVectorOp( - spaceFunction.apply(l).getSpanner(), - l, - Mutation.newInsertBuilder(op.getStaticValue("table", java.lang.String.class)) - .set(op.getStaticValue("pkey", java.lang.String.class)).to(l) - .set("VectorData").toFloat32Array(vectorF.apply(l)) - .build(), - spaceFunction.apply(l).getDbClient() - ); - } - - @Override - public GCPSpannerBaseOp getOp(long value) { - return opFunction.apply(value); - } -} diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java index c3224db1d..61a4fcdab 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java @@ -26,26 +26,47 @@ import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; +/** + * Dispenser class for creating instances of GCPSpannerUpdateDatabaseDdlOp. + */ public class GCPSpannerUpdateDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser { private static final Logger logger = LogManager.getLogger(GCPSpannerUpdateDatabaseDdlOpDispenser.class); private final LongFunction opFunction; + /** + * Constructor for GCPSpannerUpdateDatabaseDdlOpDispenser. + * + * @param adapter the GCPSpannerDriverAdapter instance + * @param op the ParsedOp instance + * @param targetFunction a LongFunction that provides the target string + */ public GCPSpannerUpdateDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction targetFunction) { super(adapter, op, targetFunction); this.opFunction = createOpFunction(op); } + /** + * Creates a LongFunction that generates GCPSpannerUpdateDatabaseDdlOp instances. + * + * @param op the ParsedOp instance + * @return a LongFunction that generates GCPSpannerUpdateDatabaseDdlOp instances + */ private LongFunction createOpFunction(ParsedOp op) { - return (l) -> new GCPSpannerUpdateDatabaseDdlOp( spaceFunction.apply(l).getSpanner(), l, - op.getAsRequiredFunction("DDL", String.class).apply(l), + targetFunction.apply(l), spaceFunction.apply(l).getDbAdminClient(), spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString()) ); } + /** + * Retrieves an operation instance based on the provided value. + * + * @param value the long value used to generate the operation + * @return a GCPSpannerBaseOp instance + */ @Override public GCPSpannerBaseOp getOp(long value) { return opFunction.apply(value); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java index 4565630b9..8e4dd8e94 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java @@ -23,6 +23,12 @@ import org.apache.logging.log4j.Logger; import java.util.function.LongFunction; +/** + * Abstract base class for GCP Spanner operations. + * This class implements the CycleOp interface and provides a template for executing operations with a Spanner client. + * + * @param the type of the request parameter + */ public abstract class GCPSpannerBaseOp implements CycleOp { protected final static Logger logger = LogManager.getLogger(GCPSpannerBaseOp.class); @@ -31,27 +37,50 @@ public abstract class GCPSpannerBaseOp implements CycleOp { protected final T request; protected final LongFunction apiCall; - public GCPSpannerBaseOp(Spanner searchIndexClient, T requestParam) { - this.spannerClient = searchIndexClient; + /** + * Constructs a new GCPSpannerBaseOp with the specified Spanner client and request parameter. + * + * @param spannerClient the Spanner client to use for operations + * @param requestParam the request parameter for the operation + */ + public GCPSpannerBaseOp(Spanner spannerClient, T requestParam) { + this.spannerClient = spannerClient; this.request = requestParam; this.apiCall = this::applyOp; } + /** + * Applies the operation for the given cycle value. + * This method logs the operation and handles any exceptions by throwing a RuntimeException. + * + * @param value the cycle value + * @return the result of the operation + */ @Override public final Object apply(long value) { logger.trace(() -> "applying op: " + this); try { Object result = applyOp(value); - return result; } catch (Exception rte) { throw new RuntimeException(rte); } } + /** + * Abstract method to be implemented by subclasses to define the specific operation logic. + * + * @param value the cycle value + * @return the result of the operation + */ public abstract Object applyOp(long value); + /** + * Returns a string representation of the GCPSpannerBaseOp. + * + * @return a string representation of the GCPSpannerBaseOp + */ @Override public String toString() { return "GCPSpannerBaseOp(" + this.request.getClass().getSimpleName() + ")"; diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java index 992bc95d7..28c0de139 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java @@ -19,10 +19,22 @@ package io.nosqlbench.adapter.gcpspanner.ops; import com.google.cloud.spanner.*; +/** + * This class represents an operation to execute a DML statement on Google Cloud Spanner. + * It extends the GCPSpannerBaseOp class and overrides the applyOp method to execute the DML statement. + */ public class GCPSpannerExecuteDmlOp extends GCPSpannerBaseOp { private final Statement statement; private final DatabaseClient dbClient; + /** + * Constructs a new GCPSpannerExecuteDmlOp. + * + * @param spanner the Spanner instance + * @param requestParam the request parameter + * @param statement the DML statement to execute + * @param dbClient the DatabaseClient to use for executing the statement + */ public GCPSpannerExecuteDmlOp(Spanner spanner, Long requestParam, Statement statement, DatabaseClient dbClient) { super(spanner, requestParam); @@ -30,6 +42,12 @@ public class GCPSpannerExecuteDmlOp extends GCPSpannerBaseOp { this.dbClient = dbClient; } + /** + * Executes the DML statement using the provided value. + * + * @param value the value to use for the operation + * @return the result of the DML execution + */ @Override public Object applyOp(long value) { try (ReadContext context = dbClient.singleUse()) { diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java similarity index 54% rename from nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java rename to nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java index a39c58454..219d49db9 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertVectorOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java @@ -23,16 +23,34 @@ import com.google.cloud.spanner.Mutation; import java.util.Collections; -public class GCPSpannerInsertVectorOp extends GCPSpannerBaseOp { +/** + * This class represents an operation to insert a vector into a Google Cloud Spanner database. + * It extends the GCPSpannerBaseOp class and provides the implementation for the applyOp method. + */ +public class GCPSpannerInsertOp extends GCPSpannerBaseOp { private final Mutation mutation; private final DatabaseClient dbClient; - public GCPSpannerInsertVectorOp(Spanner searchIndexClient, Long requestParam, Mutation mutation, DatabaseClient dbClient) { + /** + * Constructs a new GCPSpannerInsertVectorOp. + * + * @param searchIndexClient the Spanner client used to interact with the database + * @param requestParam the request parameter + * @param mutation the Mutation object representing the data to be inserted + * @param dbClient the DatabaseClient used to execute the mutation + */ + public GCPSpannerInsertOp(Spanner searchIndexClient, Long requestParam, Mutation mutation, DatabaseClient dbClient) { super(searchIndexClient, requestParam); this.mutation = mutation; this.dbClient = dbClient; } + /** + * Applies the insert operation using the provided mutation. + * + * @param value the value to be used in the operation + * @return the result of the write operation + */ @Override public Object applyOp(long value) { return dbClient.write(Collections.singletonList(mutation)); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java index a0903bc6b..651838412 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java @@ -22,11 +22,24 @@ import com.google.cloud.spanner.*; import com.google.common.collect.ImmutableList; import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; +/** + * This class represents an operation to update the database DDL (Data Definition Language) in Google Cloud Spanner. + * It extends the GCPSpannerBaseOp class and provides the implementation for applying the DDL update operation. + */ public class GCPSpannerUpdateDatabaseDdlOp extends GCPSpannerBaseOp { private final String createTableStatement; private final DatabaseAdminClient dbAdminClient; private final Database db; + /** + * Constructs a new GCPSpannerUpdateDatabaseDdlOp. + * + * @param searchIndexClient the Spanner client + * @param requestParam the request parameter + * @param createTableStatement the SQL statement to create the table + * @param dbAdminClient the DatabaseAdminClient to execute the DDL update + * @param db the Database object representing the target database + */ public GCPSpannerUpdateDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String createTableStatement, DatabaseAdminClient dbAdminClient, Database db) { super(searchIndexClient, requestParam); @@ -35,6 +48,13 @@ public class GCPSpannerUpdateDatabaseDdlOp extends GCPSpannerBaseOp { this.db = db; } + /** + * Applies the DDL update operation. + * + * @param value the value to be used in the operation + * @return the result of the operation + * @throws RuntimeException if an error occurs during the operation + */ @Override public Object applyOp(long value) { OperationFuture operation = dbAdminClient.updateDatabaseDdl( diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java index 67477d341..e55551c8f 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java @@ -31,6 +31,6 @@ package io.nosqlbench.adapter.gcpspanner.types; */ public enum GCPSpannerOpType { update_database_ddl, - insert_vector, + insert, execute_dml, } diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml new file mode 100644 index 000000000..7bac40406 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml @@ -0,0 +1,13 @@ +scenarios: + default: + execute_ddl: run driver=gcp_spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 + +# https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list +blocks: + execute_ddl: + ops: + op1: + update_database_ddl: | + CREATE VECTOR INDEX VectorsIndex ON vectors(value) + OPTIONS (distance_type = 'COSINE', tree_depth = 3, num_branches=1000, num_leaves = 1000000); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml new file mode 100644 index 000000000..48d7c0e71 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml @@ -0,0 +1,18 @@ +scenarios: + default: + execute_dml: run driver=gcp_spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file) + project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) + +bindings: + rw_key: ToString(); + test_vector_hdf5: HdfFileToFloatList("testdata/TEMPLATE(dataset).hdf5", "/test"); ToCqlVector() + validation_set_hdf5: HdfFileToIntArray("testdata/TEMPLATE(dataset).hdf5", "/neighbors") + + +blocks: + execute_dml: + ops: + op1: + execute_dml: | + SELECT * FROM vectors@{FORCE_INDEX=VectorsIndex} ORDER BY APPROX_COSINE_DISTANCE(ARRAY{test_vector_hdf5}, + value, options => JSON '{"num_leaves_to_search": 10}') LIMIT 100 diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml new file mode 100644 index 000000000..4695d4f29 --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml @@ -0,0 +1,11 @@ +scenarios: + default: + execute_ddl: run driver=gcp_spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 + +blocks: + execute_ddl: + ops: + op1: + update_database_ddl: | + CREATE TABLE vectors (keycol STRING(100),value ARRAY(vector_length=>25) NOT NULL) PRIMARY KEY(keycol) diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml new file mode 100644 index 000000000..3885f69fd --- /dev/null +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml @@ -0,0 +1,18 @@ +scenarios: + default: + insert_vector: >- + run driver=gcp_spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file) + project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) + +bindings: + rw_key: ToString(); + train_floatlist: HdfFileToFloatArray("glove-25-angular.hdf5", "/train"); + +blocks: + insert_vector: + ops: + op1: + insert_vector: "vectors" + query_params: + keycol: "{rw_key}" + value: "{train_floatlist}" diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md b/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md index ac0789357..70fc12fcc 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md @@ -1,67 +1,30 @@ # Google Spanner driver adapter -The Azure AI Search driver adapter is a NoSQLBench adapter for the `azure-aisearch` driver, a Java driver -for connecting to and performing operations on an instance of a Azure AI Search vector database. The driver is -leveraged from GitHub at https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/search/azure-search-documents/. - -## Run Commands (Remove prior to merge) - -### Create Collection Schema -``` -java -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar weaviate_vector_live weaviate_vectors.rampup dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=Glove_25 weaviatehost=letsweave-czgwdrw9.weaviate.network token_file=${workspace_loc:/nosqlbench}/local/weaviate/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:weaviate_1255,instance:vectors,vendor:weaviate_wcd" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 -``` - -### Delete Collection -``` -java -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.delete_index dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=cosine azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azure_aisearch,instance:vectors,vendor:azure_aisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 -``` - -### List Indexes -``` -java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.list_indexes dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 -``` - -### Upload Documents -``` -java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.upload_documents dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 -``` - -### Search Documents -``` -java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.search_documents dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5 -``` - +The Google Cloud Spanner driver adapter is a NoSQLBench adapter for the `gcp_spanner` driver, a Java driver +for connecting to and performing operations on an instance of a Google Cloud Spanner database. ## Activity Parameters The following parameters must be supplied to the adapter at runtime in order to successfully connect to an -instance of the [Azure AI Search database](https://learn.microsoft.com/en-us/rest/api/searchservice/?view=rest-searchservice-2024-07-01): +instance of the [Google Cloud Spanner database](https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/overview): -* `token` - In order to use the Weaviate database you must have an account. Once the account is created you can [request - an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be - provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and - referenced via the `token_file` config option pointing to the path of the file. -* `endpoint` - When a collection/index is created in the database the URI (aka endpoint) must be specified as well. The adapter will - use the default value of `localhost:8080` if none is provided at runtime. -* `api_version` - the api version to be used by the search client. Defaults to the latest service/api version supported - by the version of client SDK. +* `service_account_file` - In order to connect to a Spanner database you must have a [IAM service account](https://cloud.google.com/docs/authentication/provide-credentials-adc#service-account) +defined with the appropriate permissions associated with the adapter. Once the service account is created you can download +a file from the gcp console in JSON format that contains the credentials for the service account. This file must be provided +to the adapter at runtime. +* `project_id` - Project ID containing the Spanner database. See [Creating a project](https://cloud.google.com/resource-manager/docs/creating-managing-projects). +* `instance_id` - Spanner database's Instance ID. See [Creating an instance](https://cloud.google.com/spanner/docs/getting-started/java#create_an_instance). +* `database_id` - Spanner database's Database ID. See [Creating a database](https://cloud.google.com/spanner/docs/getting-started/java#create_a_database). +* In addition to this the environment variable `GOOGLE_APPLICATION_CREDENTIALS` must be set to the path of the service account file. ## Op Templates -The Azure AI Search adapter supports [**all basic operations**](../java/io/nosqlbench/adapter/azure-aisearch/ops) supported by the [Java -client SDK published by Azure AI Search](https://github.com/weaviate/java-client). The official Azure AI Search API reference can be -found at https://learn.microsoft.com/en-us/rest/api/searchservice/operation-groups?view=rest-searchservice-2024-07-01. - -The operations include a full-fledged support for key APIs available in the Java SDK client. -The following are a couple high level API operations. - -* Create or Update Index -* Delete Index -* List Indexes -* Upload Documents (vectors) -* (Vector) Search Documents (vectors) - +The Google Cloud Spanner adapter supports the following operations: +* `update_database_ddl` - Data Definition Language operations such as creating and dropping tables, indexes, etc. +* `execute_dml` - Data Manipulation Language operations. Read only operations are supported at this time, including queries +and vector queries. +* `insert` - Insert a single record, vector or non-vector, of data into the database. ## Examples -Check out the [full example workload available here](./activities/azure_aisearch_vectors_live.yaml). + --- diff --git a/nb-adapters/adapter-jdbc/pom.xml b/nb-adapters/adapter-jdbc/pom.xml index 23d3e9d28..630a7ab2a 100644 --- a/nb-adapters/adapter-jdbc/pom.xml +++ b/nb-adapters/adapter-jdbc/pom.xml @@ -33,18 +33,6 @@ into a PostegreSQL® compatible database leveraging HikariCP. - - - - - - - - - - - - @@ -73,38 +61,6 @@ 5.0.1 - - - - - - - - - - - - - - - - - - - - - com.google.cloud - google-cloud-spanner-jdbc - 2.22.0 - - - - com.google.cloud - google-cloud-spanner - 6.71.0 - - - diff --git a/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java b/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java index fa9b6eb47..11d06fe9c 100644 --- a/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java +++ b/nb-adapters/adapter-jdbc/src/main/java/io/nosqlbench/adapter/jdbc/optypes/JDBCOp.java @@ -57,14 +57,13 @@ public abstract class JDBCOp implements CycleOp { else { String url = jdbcSpace.getConnConfig().getJdbcUrl(); Properties props = jdbcSpace.getConnConfig().getDataSourceProperties(); - if (jdbcSpace.getConnConfig().getUsername() != null) props.put("user", jdbcSpace.getConnConfig().getUsername()); - if (jdbcSpace.getConnConfig().getPassword() != null) props.put("password", jdbcSpace.getConnConfig().getPassword()); + props.put("user", jdbcSpace.getConnConfig().getUsername()); + props.put("password", jdbcSpace.getConnConfig().getPassword()); connection = DriverManager.getConnection(url, props); } - if (connection.getMetaData().getDatabaseProductName().equals("PostgreSQL")) { - JDBCPgVector.addVectorType(connection); - } + // Register 'vector' type + JDBCPgVector.addVectorType(connection); if (LOGGER.isDebugEnabled()) { LOGGER.debug("A new JDBC connection ({}) is successfully created: {}", diff --git a/nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml b/nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml deleted file mode 100644 index 9e44873d4..000000000 --- a/nb-adapters/adapter-jdbc/src/main/resources/activities.baselinesv2/spanner.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# run driver=jdbc workload="/path/to/postgresql-keyvalue.yaml" tags="block:schema" threads=AUTO cycles=4 url="jdbc:postgresql://host:port/database" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName=insectdb sslrootcert="/path/to/postgresql_certs/root.crt" -vv --show-stacktraces -min_version: "5.17.2" - -scenarios: - default: -# drop: run driver=jdbc tags==block:drop threads===1 cycles==UNDEF url="jdbc:postgresql://host:port/" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName="pgsql" sslrootcert="/path/to/postgresql_certs/root.crt" -# schema: run driver=jdbc tags==block:schema threads===1 cycles==UNDEF url="jdbc:postgresql://host:port/" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName="pgsql" sslrootcert="/path/to/postgresql_certs/root.crt" - train: run driver=jdbc tags==block:train threads=1 cycles===100 url="jdbc:cloudspanner:/projects/gcp-lcm-project/instances/stratperf-sep-24/databases/baselines" dml_batch=1 autoCommit=false databaseName="baselines" -# testann: run driver=jdbc tags==block:testann threads=AUTO cycles===TEMPLATE(main-cycles,1000) url="jdbc:postgresql://host:port/" databaseName="defaultdb" portNumber=5432 user="newuser" password="CHANGE_ME" sslmode="prefer" serverName="pgsql" sslrootcert="/path/to/postgresql_certs/root.crt" - -# "jdbc:cloudspanner:/projects/gcp-lcm-project/instances/stratperf-sep-24/databases/baselines?credentials=/home/cloudspanner-keys/my-key.json;autocommit=false"; - -bindings: - rw_key: ToString(); - train_floatlist: HdfFileToFloatList("/home/mwolters138/datasets/glove-25-angular.hdf5", "/train"); ToCqlVector(); - test_floatlist: HdfFileToFloatList("/home/mwolters138/datasets/glove-25-angular.hdf5", "/test"); ToCqlVector(); - relevant_indices: HdfFileToIntArray("/home/mwolters138/datasets/glove-25-angular.hdf5", "/neighbors") - -blocks: -# drop: -# ops: -# drop_vector_index: -# ddl: | -# DROP INDEX IF EXISTS idx_TEMPLATE(tablename,baseline)_TEMPLATE(indextype)_TEMPLATE(similarity_function); -# drop_table: -# ddl: | -# DROP TABLE IF EXISTS TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline); -# ## -# # NOTE: Do NOT enable this block for 'runall.sh' script -# # -------------------------------------------------- -# # drop_schema: -# # ddl: | -# # DROP SCHEMA IF EXISTS TEMPLATE(schemaname,public); -# -# schema: -# ops: -# create_schema: -# ddl: | -# CREATE SCHEMA IF NOT EXISTS TEMPLATE(schemaname,public); -# create_table: -# ddl: | -# CREATE TABLE IF NOT EXISTS TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) -# (key TEXT PRIMARY KEY, value vector(TEMPLATE(dimensions,5))); -# create_vector_index: -# ddl: | -# CREATE INDEX IF NOT EXISTS idx_TEMPLATE(tablename,baseline)_TEMPLATE(indextype)_TEMPLATE(similarity_function) -# ON TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) -# USING TEMPLATE(indextype) (value vector_TEMPLATE(similarity_function)_ops) -# WITH (TEMPLATE(indexopt)); - - train: - params: - prepared: true - ops: - main_insert: - dmlwrite: | - INSERT INTO TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) VALUES (?,?) - ON CONFLICT DO NOTHING; - prep_stmt_val_arr: | - {rw_key},{train_floatlist} - -# testann: -# params: -# prepared: true -# ops: -# # NOTE: right now this is only for cosine similarity. -# # in baselinetor, '<=>' is for cosine similarity -# # '<->' is for euclidean distance -# # '<#>' is for inner product -# main_select: -# dmlread: | -# SELECT * -# FROM TEMPLATE(schemaname,public).TEMPLATE(tablename,baseline) -# ORDER BY value <=> ? -# LIMIT TEMPLATE(top_k,100); -# prep_stmt_val_arr: | -# {test_floatlist} -# ################################# -# ## NOTE: -# # 1). The script blocks below are ONLY relevant with Vector relevancy score verification -# # 2). The "verifier-key" must match the Vector data identifier column name (e.g. primary key name) -# # right now the identifier must be a type that can be converted to int. -# verifier-key: "key" -# verifier-init: | -# relevancy=scriptingmetrics.newRelevancyMeasures(_parsed_op); -# k=TEMPLATE(top_k,100) -# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.recall("recall",k)); -# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.precision("precision",k)); -# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.F1("F1",k)); -# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.reciprocal_rank("RR",k)); -# relevancy.addFunction(io.nosqlbench.engine.extensions.computefunctions.RelevancyFunctions.average_precision("AP",k)); -# verifier: | -# // driver-specific function -# actual_indices=pgvec_utils.getValueListForVerifierKey(result); -# // driver-agnostic function -# relevancy.accept({relevant_indices},actual_indices); -# // because we are "verifying" although this needs to be reorganized -# return true; diff --git a/nb-adapters/nb-adapters-included/pom.xml b/nb-adapters/nb-adapters-included/pom.xml index e812a1384..7c854e88e 100644 --- a/nb-adapters/nb-adapters-included/pom.xml +++ b/nb-adapters/nb-adapters-included/pom.xml @@ -74,7 +74,7 @@ adapter-cqld4-include - false + true @@ -102,7 +102,7 @@ adapter-http-include - false + true @@ -130,7 +130,7 @@ adapter-tcp-include - false + true @@ -144,7 +144,7 @@ adapter-dataapi-include - false + true @@ -158,7 +158,7 @@ adapter-dynamodb-include - false + true @@ -172,7 +172,7 @@ adapter-mongodb-include - false + true @@ -186,7 +186,7 @@ adapter-pulsar-include - false + true @@ -200,7 +200,7 @@ adapter-s4j-include - false + true @@ -214,7 +214,7 @@ adapter-neo4j-include - false + true @@ -228,7 +228,7 @@ adapter-kafka-include - false + true @@ -242,7 +242,7 @@ adapter-amqp-include - false + true @@ -256,7 +256,7 @@ adapter-qdrant-include - false + true @@ -270,7 +270,7 @@ adapter-weaviate-include - false + true @@ -284,7 +284,7 @@ adapter-azure-aisearch-include - false + true diff --git a/nb-adapters/pom.xml b/nb-adapters/pom.xml index d5dd20643..fc27812a8 100644 --- a/nb-adapters/pom.xml +++ b/nb-adapters/pom.xml @@ -57,7 +57,7 @@ adapter-cqld4-module - false + true adapter-cqld4 @@ -77,7 +77,7 @@ adapter-http-module - false + true adapter-http @@ -97,7 +97,7 @@ adapter-tcp-module - false + true adapter-tcp @@ -107,7 +107,7 @@ adapter-dynamodb-module - false + true adapter-dynamodb @@ -117,7 +117,7 @@ adapter-mongodb-module - false + true adapter-mongodb @@ -127,7 +127,7 @@ adapter-neo4j-module - false + true adapter-neo4j @@ -137,7 +137,7 @@ adapter-pulsar-module - false + true adapter-pulsar @@ -147,7 +147,7 @@ adapter-s4j-module - false + true adapter-s4j @@ -157,7 +157,7 @@ adapter-kafka-module - false + true adapter-kafka @@ -167,7 +167,7 @@ adapter-amqp-module - false + true adapter-amqp @@ -177,7 +177,7 @@ adapter-dataapi-module - false + true adapter-dataapi @@ -187,7 +187,7 @@ adapter-qdrant-module - false + true adapter-qdrant @@ -197,7 +197,7 @@ adapter-weaviate-module - false + true adapter-weaviate @@ -207,7 +207,7 @@ adapter-azure-aisearch-module - false + true adapter-azure-aisearch From bad242f5779d99295ad9e3657950baad4de26747 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 1 Oct 2024 13:29:42 -0400 Subject: [PATCH 7/8] rename to simply spanner,added check for service account file --- .../adapter/gcpspanner/GCPSpannerAdapterUtils.java | 2 +- .../nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java | 9 ++++----- .../src/main/resources/activities/execute_index_ddl.yaml | 2 +- .../resources/activities/execute_queryvector_dml.yaml | 2 +- .../src/main/resources/activities/execute_table_ddl.yaml | 2 +- .../src/main/resources/activities/insert_vector.yaml | 2 +- .../src/main/resources/{gcp-spanner.md => spanner.md} | 0 7 files changed, 9 insertions(+), 10 deletions(-) rename nb-adapters/adapter-gcp-spanner/src/main/resources/{gcp-spanner.md => spanner.md} (100%) diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java index 2b65059ab..c68e0e5a2 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.List; public class GCPSpannerAdapterUtils { - public static final String SPANNER = "gcp_spanner"; + public static final String SPANNER = "spanner"; public static int[] getKeyArrayFromResultSet(ResultSet rs) { List values = new ArrayList<>(); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java index 09d0d92b9..802ac81d3 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java @@ -33,9 +33,6 @@ import org.apache.logging.log4j.Logger; * @see DML Syntax * @see spanner rpc api calls * @see SQL functionality related to vector indices - * @see - * @see - * @see */ public class GCPSpannerSpace implements AutoCloseable { private final static Logger logger = LogManager.getLogger(GCPSpannerSpace.class); @@ -89,8 +86,10 @@ public class GCPSpannerSpace implements AutoCloseable { if ( cfg.getOptional("database_id").isEmpty() || cfg.getOptional("project_id").isEmpty() || - cfg.getOptional("instance_id").isEmpty()) { - throw new RuntimeException("You must provide all 'service_account_file', 'project_id', 'instance_id' & 'database_id' to configure a Google Spanner client"); + cfg.getOptional("instance_id").isEmpty() || + cfg.getOptional("service_account_file").isEmpty()) { + throw new RuntimeException("You must provide all 'service_account_file', 'project_id', 'instance_id' &" + + " 'database_id' to configure a Google Spanner client"); } String projectId = cfg.get("project_id"); String instanceId = cfg.get("instance_id"); diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml index 7bac40406..e7f67069d 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=gcp_spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 # https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml index 48d7c0e71..9357e4b2f 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_dml: run driver=gcp_spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file) + execute_dml: run driver=spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml index 4695d4f29..5a33d38a4 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml @@ -1,6 +1,6 @@ scenarios: default: - execute_ddl: run driver=gcp_spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) + execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1 blocks: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml index 3885f69fd..d15c1bce3 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml +++ b/nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml @@ -1,7 +1,7 @@ scenarios: default: insert_vector: >- - run driver=gcp_spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file) + run driver=spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file) project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles) bindings: diff --git a/nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md b/nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md similarity index 100% rename from nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md rename to nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md From a2d74775eb5847da5344c8ca65e041a1776c573e Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 1 Oct 2024 13:52:29 -0400 Subject: [PATCH 8/8] spanner version 6.74->6.76 and updated copyright date --- mvn-defaults/pom.xml | 2 +- .../nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java | 2 +- .../nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java | 2 +- .../gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java | 2 +- .../opdispensers/GCPSpannerExecuteDmlOpDispenser.java | 2 +- .../gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java | 2 +- .../opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java | 2 +- .../adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java | 2 +- .../nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java | 2 +- .../adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index e42d93c0e..94fcf29e3 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -418,7 +418,7 @@ com.google.cloud google-cloud-spanner - 6.74.0 + 6.76.0 diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java index c68e0e5a2..821aa6009 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java index 97aef8007..97c789fa7 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java index e7550a3a0..b3c40b0e3 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerBaseOpDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java index 9c2fd0d83..ab76d2aac 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerExecuteDmlOpDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java index 299a0a642..adcbc812e 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerInsertOpDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java index 61a4fcdab..6d466410a 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerUpdateDatabaseDdlOpDispenser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java index 28c0de139..0168f8931 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerExecuteDmlOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java index 219d49db9..f45120753 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerInsertOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java index 651838412..7e9de0c89 100644 --- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java +++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerUpdateDatabaseDdlOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 nosqlbench + * Copyright (c) 2020-2024 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.