mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
pinecone adapter updates for vector branch
This commit is contained in:
Jonathan Shook
parent
bbc5844671
commit
f53f52efa4
@ -19,10 +19,10 @@ package io.nosqlbench.adapter.pinecone;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.api.config.standard.NBConfigModel;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.BaseDriverAdapter;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.nb.annotations.Service;
|
||||
|
||||
import java.util.function.Function;
|
||||
|
||||
@ -20,10 +20,10 @@ import io.nosqlbench.adapter.pinecone.opdispensers.*;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOpTypes;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.adapters.api.activityimpl.OpMapper;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.nosqlbench.engine.api.templating.TypeAndTarget;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
@ -21,7 +21,7 @@ import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeDeleteOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.DeleteRequest;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
@ -21,7 +21,7 @@ import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeDescribeIndexStatsOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.DescribeIndexStatsRequest;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
@ -20,7 +20,7 @@ import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeFetchOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.FetchRequest;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
@ -22,8 +22,8 @@ import com.google.protobuf.Value;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
||||
@ -21,7 +21,7 @@ import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeQueryOp;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.QueryRequest;
|
||||
import io.pinecone.proto.QueryVector;
|
||||
import io.pinecone.proto.SparseValues;
|
||||
|
||||
@ -21,7 +21,7 @@ import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeUpdateOp;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.SparseValues;
|
||||
import io.pinecone.proto.UpdateRequest;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
|
||||
@ -21,7 +21,7 @@ import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||
import io.nosqlbench.adapter.pinecone.ops.PineconeUpsertOp;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.SparseValues;
|
||||
import io.pinecone.proto.UpsertRequest;
|
||||
import io.pinecone.proto.Vector;
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.PineconeConnection;
|
||||
import io.pinecone.proto.DeleteRequest;
|
||||
import io.pinecone.proto.DeleteResponse;
|
||||
@ -41,8 +41,9 @@ public class PineconeDeleteOp extends PineconeOp {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public Object apply(long value) {
|
||||
DeleteResponse response = connection.getBlockingStub().delete(request);
|
||||
logger.debug("Pinecone delete request successful: " + response.toString());
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.DescribeIndexStatsRequest;
|
||||
import io.pinecone.proto.DescribeIndexStatsResponse;
|
||||
import io.pinecone.PineconeConnection;
|
||||
@ -43,8 +43,8 @@ public class PineconeDescribeIndexStatsOp extends PineconeOp {
|
||||
this.request = request;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
@Override
|
||||
public Object apply(long value) {
|
||||
DescribeIndexStatsResponse response = connection.getBlockingStub().describeIndexStats(request);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Vector counts:");
|
||||
@ -52,6 +52,6 @@ public class PineconeDescribeIndexStatsOp extends PineconeOp {
|
||||
logger.debug(namespace.getKey() + ": " + namespace.getValue().getVectorCount());
|
||||
}
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.FetchRequest;
|
||||
import io.pinecone.PineconeConnection;
|
||||
import io.pinecone.proto.FetchResponse;
|
||||
@ -44,12 +44,13 @@ public class PineconeFetchOp extends PineconeOp {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public Object apply(long value) {
|
||||
FetchResponse response = connection.getBlockingStub().fetch(request);
|
||||
if (logger.isDebugEnabled()) {
|
||||
for (Map.Entry<String, Vector> vectors: response.getVectorsMap().entrySet()) {
|
||||
logger.debug(vectors.getKey() + ": " + vectors.getValue().toString());
|
||||
}
|
||||
}
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,10 +16,10 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.RunnableOp;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
|
||||
import io.pinecone.PineconeConnection;
|
||||
|
||||
public abstract class PineconeOp implements RunnableOp {
|
||||
public abstract class PineconeOp implements CycleOp {
|
||||
protected final PineconeConnection connection;
|
||||
|
||||
public PineconeOp(PineconeConnection connection) {
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.QueryRequest;
|
||||
import io.pinecone.PineconeConnection;
|
||||
import io.pinecone.proto.QueryResponse;
|
||||
@ -43,7 +43,7 @@ public class PineconeQueryOp extends PineconeOp {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public Object apply(long value) {
|
||||
QueryResponse response = connection.getBlockingStub().query(request);
|
||||
if (logger.isDebugEnabled()) {
|
||||
for (ScoredVector scored : response.getMatchesList()) {
|
||||
@ -55,5 +55,6 @@ public class PineconeQueryOp extends PineconeOp {
|
||||
}
|
||||
}
|
||||
}
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.UpdateRequest;
|
||||
import io.pinecone.PineconeConnection;
|
||||
import io.pinecone.proto.UpdateResponse;
|
||||
@ -41,8 +41,9 @@ public class PineconeUpdateOp extends PineconeOp {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public Object apply(long value) {
|
||||
UpdateResponse response = connection.getBlockingStub().update(request);
|
||||
logger.debug("UpdateResponse successful: " + response.toString());
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
package io.nosqlbench.adapter.pinecone.ops;
|
||||
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import io.pinecone.proto.UpsertRequest;
|
||||
import io.pinecone.PineconeConnection;
|
||||
import io.pinecone.proto.UpsertResponse;
|
||||
@ -41,8 +41,10 @@ public class PineconeUpsertOp extends PineconeOp {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
public Object apply(long value) {
|
||||
UpsertResponse response = connection.getBlockingStub().upsert(request);
|
||||
logger.debug("Put " + response.getUpsertedCount() + " vectors into the index");
|
||||
return response;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,36 @@
|
||||
scenarios:
|
||||
default:
|
||||
mixed: run driver=pinecone cycles=1000 apiKey=2f55b2f0-670f-4c51-9073-4d37142b761a projectName=a850334 environment=us-east-1-aws tags='block:main-.*'
|
||||
|
||||
bindings:
|
||||
id: Mod(<<keycount:1000000000>>); ToString() -> String
|
||||
vector_value: CircleVectors(100000, "io.nosqlbench.virtdata.library.basics.shared.vectors.algorithms.GoldenAngle")
|
||||
state: StateCodes()
|
||||
|
||||
blocks:
|
||||
# main-write:
|
||||
# params:
|
||||
# ratio: 1
|
||||
# ops:
|
||||
# op1:
|
||||
# upsert: "circles"
|
||||
# namespace: "example_namespace"
|
||||
# upsert_vectors:
|
||||
# - id: "{id}"
|
||||
# values: "{vector_value}"
|
||||
# metadata:
|
||||
# state: "{state}"
|
||||
main-read:
|
||||
params:
|
||||
ratio: 1
|
||||
ops:
|
||||
op1:
|
||||
query: "circles"
|
||||
vector: "{vector_value}"
|
||||
namespace: "example_namespace"
|
||||
top_k: 1
|
||||
include_values: true
|
||||
include_metadata: true
|
||||
#result should be type QueryResponse
|
||||
verifier: |
|
||||
result.getMatchesList().get(0).getValuesList()=={vector_value}
|
||||
@ -20,13 +20,13 @@ import io.nosqlbench.adapter.pinecone.opdispensers.*;
|
||||
import io.nosqlbench.adapter.pinecone.ops.*;
|
||||
import io.nosqlbench.api.config.NBLabeledElement;
|
||||
import io.nosqlbench.api.config.standard.NBConfiguration;
|
||||
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.engine.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.engine.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||
import io.nosqlbench.adapters.api.activityconfig.OpsLoader;
|
||||
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplate;
|
||||
import io.nosqlbench.adapters.api.activityconfig.yaml.OpTemplateFormat;
|
||||
import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
|
||||
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
|
||||
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache;
|
||||
import io.nosqlbench.adapters.api.templating.ParsedOp;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user