IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-01-10 07:53:08 -06:00
Code Issues Packages Projects Releases Wiki Activity

Initial skeleton for Spanner

Browse Source
This commit is contained in:
Madhavan Sridharan 2024-09-12 14:46:21 -04:00
parent ee271124f5
commit 41fd1b283b
12 changed files with 578 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
mvn-defaults/pom.xml
View File

@ -415,6 +415,12 @@
<version>1.13.2</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-spanner</artifactId>
<version>6.74.0</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -438,7 +444,7 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>

55
nb-adapters/adapter-gcp-spanner/pom.xml Normal file
View File

@ -0,0 +1,55 @@
<!--
~ Copyright (c) 2020-2024 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!--<?xml version="1.0" encoding="UTF-8"?>-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>adapter-gcp-spanner</artifactId>
<packaging>jar</packaging>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>${revision}</version>
<relativePath>../../mvn-defaults</relativePath>
</parent>
<name>${project.artifactId}</name>
<description>
A NoSQLBbench driver adapter module for the Google Spanner database.
</description>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-annotations</artifactId>
<version>${revision}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapters-api</artifactId>
<version>${revision}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-spanner</artifactId>
</dependency>
</dependencies>
</project>

21
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerAdapterUtils.java Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner;
public class GCPSpannerAdapterUtils {
public static final String SPANNER = "gcp_spanner";
}

55
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapter.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.labels.NBLabels;
import java.util.function.Function;
import static io.nosqlbench.adapter.gcpspanner.GCPSpannerAdapterUtils.SPANNER;
@Service(value = DriverAdapter.class, selector = SPANNER)
public class GCPSpannerDriverAdapter extends BaseDriverAdapter<GCPSpannerBaseOp<?>, GCPSpannerSpace> {
public GCPSpannerDriverAdapter(NBComponent parentComponent, NBLabels labels) {
super(parentComponent, labels);
}
@Override
public OpMapper<GCPSpannerBaseOp<?>> getOpMapper() {
return new GCPSpannerOpMapper(this);
}
@Override
public Function<String, ? extends GCPSpannerSpace> getSpaceInitializer(NBConfiguration cfg) {
return (s) -> new GCPSpannerSpace(s, cfg);
}
@Override
public NBConfigModel getConfigModel() {
return super.getConfigModel().add(GCPSpannerSpace.getConfigModel());
}
}

32
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerDriverAdapterLoader.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner;
import io.nosqlbench.adapter.diag.DriverAdapterLoader;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.labels.NBLabels;
import static io.nosqlbench.adapter.gcpspanner.GCPSpannerAdapterUtils.SPANNER;
@Service(value = DriverAdapterLoader.class, selector = SPANNER)
public class GCPSpannerDriverAdapterLoader implements DriverAdapterLoader {
@Override
public GCPSpannerDriverAdapter load(NBComponent parent, NBLabels childLabels) {
return new GCPSpannerDriverAdapter(parent, childLabels);
}
}

68
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
import io.nosqlbench.adapter.gcpspanner.types.GCPSpannerOpType;
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.engine.api.templating.TypeAndTarget;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class GCPSpannerOpMapper implements OpMapper<GCPSpannerBaseOp<?>> {
private static final Logger logger = LogManager.getLogger(GCPSpannerOpMapper.class);
private final GCPSpannerDriverAdapter adapter;
/**
* Create a new {@code GCPSpannerOpMapper} implementing the {@link OpMapper}.
* interface.
*
* @param adapter The associated {@link GCPSpannerDriverAdapter}
*/
public GCPSpannerOpMapper(GCPSpannerDriverAdapter adapter) {
this.adapter = adapter;
}
/**
* Given an instance of a {@link ParsedOp} returns the appropriate
* {@link GCPSpannerBaseOpDispenser} subclass.
*
* @param op The {@link ParsedOp} to be evaluated
* @return The correct {@link GCPSpannerBaseOpDispenser} subclass based on
* the op type
*/
@Override
public OpDispenser<? extends GCPSpannerBaseOp<?>> apply(ParsedOp op) {
TypeAndTarget<GCPSpannerOpType, String> typeAndTarget = op.getTypeAndTarget(GCPSpannerOpType.class,
String.class, "type", "target");
logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'");
return switch (typeAndTarget.enumId) {
// case delete_index -> new GCPSpannerDeleteIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
case create_table ->
new GCPSpannerCreateTableOpDispenser(adapter, op, typeAndTarget.targetFunction);
// case list_indexes -> new GCPSpannerListIndexesOpDispenser(adapter, op, typeAndTarget.targetFunction);
// case upload_documents -> new GCPSpannerUploadDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction);
// case search_documents -> new GCPSpannerSearchDocumentsOpDispenser(adapter, op, typeAndTarget.targetFunction);
// default -> throw new RuntimeException(
// "Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + "mapping parsed op " + op);
};
}
}

132
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerSpace.java Normal file
View File

@ -0,0 +1,132 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner;
import com.google.cloud.spanner.*;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfigModel;
import io.nosqlbench.nb.api.config.standard.NBConfiguration;
import io.nosqlbench.nb.api.config.standard.Param;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
* @see <a href="https://cloud.google.com/spanner/docs/find-approximate-nearest-neighbors">ANN Docs</a>
* @see <a href="https://github.com/googleapis/java-spanner">Spanner Java Client</a>
* @see <a href="https://cloud.google.com/spanner/docs/getting-started/java">Getting started in Java</a>
* @see <a href="https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow">Authentication methods at Google</a>
* @see <a href="https://cloud.google.com/java/docs/reference/google-cloud-spanner/latest/overview">Library Reference Doc</a>
* @see <a href="https://cloud.google.com/spanner/docs/reference/standard-sql/dml-syntax">DML Syntax</a>
* @see <a href=""></a>
* @see <a href=""></a>
* @see <a href=""></a>
* @see <a href=""></a>
* @see <a href=""></a>
*/
public class GCPSpannerSpace implements AutoCloseable {
private final static Logger logger = LogManager.getLogger(GCPSpannerSpace.class);
private final String name;
private final NBConfiguration cfg;
protected Spanner spanner;
protected DatabaseAdminClient dbAdminClient;
protected DatabaseClient dbClient;
/**
* Create a new {@code GCPSpannerSpace} Object which stores all stateful
* contextual information needed to interact with the <b>Google Spanner</b>
* database instance.
*
* @param name The name of this space
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
*/
public GCPSpannerSpace(String name, NBConfiguration cfg) {
this.name = name;
this.cfg = cfg;
}
public synchronized Spanner getSpanner() {
if (spanner == null) {
spanner = createSpanner();
}
return spanner;
}
public synchronized DatabaseAdminClient getDbAdminClient() {
return dbAdminClient;
}
public synchronized DatabaseClient getDbClient() {
return dbClient;
}
private Spanner createSpanner() {
if (/*cfg.getOptional("service_account_file").isEmpty() ||*/
cfg.getOptional("database_id").isEmpty() ||
cfg.getOptional("project_id").isEmpty() ||
cfg.getOptional("instance_id").isEmpty()) {
throw new RuntimeException("You must provide all 'service_account_file', 'project_id', 'instance_id' & 'database_id' to configure a Google Spanner client");
}
String projectId = cfg.get("project_id");
String instanceId = cfg.get("instance_id");
String databaseId = cfg.get("database_id");
var spannerClient = SpannerOptions.newBuilder().setProjectId(projectId).build().getService();
dbAdminClient = spannerClient.getDatabaseAdminClient();
dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId));
// var requiredToken = cfg.getOptional("token_file").map(Paths::get).map(tokenFilePath -> {
// try {
// return Files.readAllLines(tokenFilePath).getFirst();
// } catch (IOException e) {
// String error = "Error while reading token from file:" + tokenFilePath;
// logger.error(error, e);
// throw new RuntimeException(e);
// }
// }).orElseGet(() -> cfg.getOptional("token").orElseThrow(() -> new RuntimeException(
// "You must provide either a 'token_file' or a 'token' to configure a Azure AI Search client")));
// logger.info(() -> "Creating new Azure AI Search Client with (masked) token/key ["
// + GCPSpannerAdapterUtils.maskDigits(requiredToken) + "], uri/endpoint [" + uri + "]");
//
// var spannerBuilder = SpannerOptions().endpoint(uri);
// if (!requiredToken.isBlank()) {
// SpannerBuilder = SpannerBuilder.credential(new AzureKeyCredential(requiredToken));
// } else {
// TokenCredential tokenCredential = new DefaultAzureCredentialBuilder().build();
// SpannerBuilder = SpannerBuilder.credential(tokenCredential);
// }
// // Should we leave these below to leverage the SearchServiceVersion.getLatest()?
// String apiVersion = cfg.getOptional("api_version").orElse(SearchServiceVersion.V2024_07_01.name());
// logger.warn(
// () -> "Latest search service version supported by this client is '" + SearchServiceVersion.getLatest()
// + "', but we're using '" + apiVersion + "' version. Ignore this warning if both are same.");
return spannerClient;
}
public static NBConfigModel getConfigModel() {
return ConfigModel.of(GCPSpannerSpace.class)
.add(Param.optional("service_account_file", String.class, "the file to load the api token/key from. See https://cloud.google.com/docs/authentication/provide-credentials-adc#service-account"))
// .add(Param.defaultTo("token", "my-spanner-admin-key-changeme")
// .setDescription("the Spanner api token/key to use to connect to the database"))
.add(Param.optional("project_id", String.class,"Project ID containing the Spanner database. See https://cloud.google.com/resource-manager/docs/creating-managing-projects"))
.add(Param.optional("instance_id", String.class, "Spanner database's Instance ID containing. See https://cloud.google.com/spanner/docs/getting-started/java#create_an_instance"))
.add(Param.optional("database_id", String.class, "Spanner Database ID. See https://cloud.google.com/spanner/docs/getting-started/java#create_a_database"))
.asReadOnly();
}
@Override
public void close() throws Exception {
spanner = null;
}
}

65
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerBaseOp.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner.ops;
import com.google.cloud.spanner.Spanner;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
public abstract class GCPSpannerBaseOp<T> implements CycleOp<Object> {
protected final static Logger logger = LogManager.getLogger(GCPSpannerBaseOp.class);
protected final Spanner spannerClient;
protected final T request;
protected final LongFunction<Object> apiCall;
public GCPSpannerBaseOp(Spanner searchIndexClient, T requestParam) {
this.spannerClient = searchIndexClient;
this.request = requestParam;
this.apiCall = this::applyOp;
}
public GCPSpannerBaseOp(Spanner spanner, T requestParam, LongFunction<Object> call) {
this.spannerClient = spanner;
this.request = requestParam;
this.apiCall = call;
}
@Override
public final Object apply(long value) {
logger.trace(() -> "applying op: " + this);
try {
Object result = applyOp(value);
return result;
} catch (Exception rte) {
throw new RuntimeException(rte);
}
};
public abstract Object applyOp(long value);
@Override
public String toString() {
return "GCPSpannerBaseOp(" + this.request.getClass().getSimpleName() + ")";
}
}

24
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner.types;
/**
* https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_statements
*/
public enum GCPSpannerOpType {
create_table,
}

67
nb-adapters/adapter-gcp-spanner/src/main/resources/gcp-spanner.md Normal file
View File

@ -0,0 +1,67 @@
# Google Spanner driver adapter
The Azure AI Search driver adapter is a NoSQLBench adapter for the `azure-aisearch` driver, a Java driver
for connecting to and performing operations on an instance of a Azure AI Search vector database. The driver is
leveraged from GitHub at https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/search/azure-search-documents/.
## Run Commands (Remove prior to merge)
### Create Collection Schema
```
java -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar weaviate_vector_live weaviate_vectors.rampup dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=Glove_25 weaviatehost=letsweave-czgwdrw9.weaviate.network token_file=${workspace_loc:/nosqlbench}/local/weaviate/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:weaviate_1255,instance:vectors,vendor:weaviate_wcd" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5
```
### Delete Collection
```
java -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.delete_index dimensions=25 testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 similarity_function=cosine azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azure_aisearch,instance:vectors,vendor:azure_aisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5
```
### List Indexes
```
java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.list_indexes dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5
```
### Upload Documents
```
java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.upload_documents dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5
```
### Search Documents
```
java --enable-preview -jar ${workspace_loc:/nosqlbench}/nb5/target/nb5.jar azure_aisearch_vectors_live azure_aisearch_vectors.search_documents dimensions=25 similarity_function=cosine testsize=10000 trainsize=1183514 dataset=glove-25-angular filetype=hdf5 collection=glove_25 azureaisearchhost=https://stratperf-aisearch-central-india-free-tier.search.windows.net token_file=${workspace_loc:/nosqlbench}/local/azure_aisearch/apikey --progress console:1s -v --add-labels "dimensions:25,dataset=glove-25" --add-labels="target:azureaisearch,instance:vectors,vendor:azureaisearch" --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators "[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]" --report-interval 10 --show-stacktraces --logs-max 5
```
## Activity Parameters
The following parameters must be supplied to the adapter at runtime in order to successfully connect to an
instance of the [Azure AI Search database](https://learn.microsoft.com/en-us/rest/api/searchservice/?view=rest-searchservice-2024-07-01):
* `token` - In order to use the Weaviate database you must have an account. Once the account is created you can [request
an api key/token](https://weaviate.io/developers/wcs/quickstart#explore-the-details-panel). This key will need to be
provided any time a database connection is desired. Alternatively, the api key can be stored in a file securely and
referenced via the `token_file` config option pointing to the path of the file.
* `endpoint` - When a collection/index is created in the database the URI (aka endpoint) must be specified as well. The adapter will
use the default value of `localhost:8080` if none is provided at runtime.
* `api_version` - the api version to be used by the search client. Defaults to the latest service/api version supported
by the version of client SDK.
## Op Templates
The Azure AI Search adapter supports [**all basic operations**](../java/io/nosqlbench/adapter/azure-aisearch/ops) supported by the [Java
client SDK published by Azure AI Search](https://github.com/weaviate/java-client). The official Azure AI Search API reference can be
found at https://learn.microsoft.com/en-us/rest/api/searchservice/operation-groups?view=rest-searchservice-2024-07-01.
The operations include a full-fledged support for key APIs available in the Java SDK client.
The following are a couple high level API operations.
* Create or Update Index
* Delete Index
* List Indexes
* Upload Documents (vectors)
* (Vector) Search Documents (vectors)
## Examples
Check out the [full example workload available here](./activities/azure_aisearch_vectors_live.yaml).
---

42
nb-adapters/nb-adapters-included/pom.xml
View File

@ -74,7 +74,7 @@
<profile>
<id>adapter-cqld4-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -102,7 +102,7 @@
<profile>
<id>adapter-http-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -130,7 +130,7 @@
<profile>
<id>adapter-tcp-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -144,7 +144,7 @@
<profile>
<id>adapter-dataapi-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -158,7 +158,7 @@
<profile>
<id>adapter-dynamodb-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -172,7 +172,7 @@
<profile>
<id>adapter-mongodb-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -186,7 +186,7 @@
<profile>
<id>adapter-pulsar-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -200,7 +200,7 @@
<profile>
<id>adapter-s4j-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -214,7 +214,7 @@
<profile>
<id>adapter-neo4j-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -228,7 +228,7 @@
<profile>
<id>adapter-kafka-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -242,7 +242,7 @@
<profile>
<id>adapter-amqp-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -256,7 +256,7 @@
<profile>
<id>adapter-qdrant-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -270,7 +270,7 @@
<profile>
<id>adapter-weaviate-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -284,7 +284,7 @@
<profile>
<id>adapter-azure-aisearch-include</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
@ -295,6 +295,20 @@
</dependencies>
</profile>
<profile>
<id>adapter-gcp-spanner-include</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-gcp-spanner</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
</profiles>
</project>

38
nb-adapters/pom.xml
View File

@ -57,7 +57,7 @@
<profile>
<id>adapter-cqld4-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-cqld4</module>
@ -77,7 +77,7 @@
<profile>
<id>adapter-http-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-http</module>
@ -97,7 +97,7 @@
<profile>
<id>adapter-tcp-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-tcp</module>
@ -107,7 +107,7 @@
<profile>
<id>adapter-dynamodb-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-dynamodb</module>
@ -117,7 +117,7 @@
<profile>
<id>adapter-mongodb-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-mongodb</module>
@ -127,7 +127,7 @@
<profile>
<id>adapter-neo4j-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-neo4j</module>
@ -137,7 +137,7 @@
<profile>
<id>adapter-pulsar-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-pulsar</module>
@ -147,7 +147,7 @@
<profile>
<id>adapter-s4j-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-s4j</module>
@ -157,7 +157,7 @@
<profile>
<id>adapter-kafka-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-kafka</module>
@ -167,7 +167,7 @@
<profile>
<id>adapter-amqp-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-amqp</module>
@ -177,7 +177,7 @@
<profile>
<id>adapter-dataapi-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-dataapi</module>
@ -187,7 +187,7 @@
<profile>
<id>adapter-qdrant-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-qdrant</module>
@ -197,7 +197,7 @@
<profile>
<id>adapter-weaviate-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-weaviate</module>
@ -207,12 +207,22 @@
<profile>
<id>adapter-azure-aisearch-module</id>
<activation>
<activeByDefault>true</activeByDefault>
<activeByDefault>false</activeByDefault>
</activation>
<modules>
<module>adapter-azure-aisearch</module>
</modules>
</profile>
<profile>
<id>adapter-gcp-spanner-module</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<modules>
<module>adapter-gcp-spanner</module>
</modules>
</profile>
</profiles>
</project>