mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
adding comments
This commit is contained in:
parent
3c9cae7f90
commit
199bc58a90
@ -20,18 +20,33 @@ public class PineconeOpMapper implements OpMapper<PineconeOp> {
|
|||||||
private final DriverSpaceCache<? extends PineconeSpace> spaceCache;
|
private final DriverSpaceCache<? extends PineconeSpace> spaceCache;
|
||||||
private final NBConfiguration cfg;
|
private final NBConfiguration cfg;
|
||||||
|
|
||||||
public PineconeOpMapper(PineconeDriverAdapter adapter, DriverSpaceCache<? extends PineconeSpace> spaceCache, NBConfiguration cfg) {
|
/**
|
||||||
|
* Create a new PineconeOpMapper implementing the {@link OpMapper} interface.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param spaceCache A cached context Object of thpe {@link PineconeSpace})
|
||||||
|
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
|
||||||
|
*/
|
||||||
|
public PineconeOpMapper(PineconeDriverAdapter adapter,
|
||||||
|
DriverSpaceCache<? extends PineconeSpace> spaceCache,
|
||||||
|
NBConfiguration cfg) {
|
||||||
this.adapter = adapter;
|
this.adapter = adapter;
|
||||||
this.spaceCache = spaceCache;
|
this.spaceCache = spaceCache;
|
||||||
this.cfg = cfg;
|
this.cfg = cfg;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given an instance of a {@link ParsedOp} returns the appropriate {@link PineconeOpDispenser} subclass
|
||||||
|
*
|
||||||
|
* @param op The ParsedOp to be evaluated
|
||||||
|
* @return The correct PineconeOpDispenser subclass based on the op type
|
||||||
|
*/
|
||||||
@Override
|
@Override
|
||||||
public OpDispenser<? extends PineconeOp> apply(ParsedOp op) {
|
public OpDispenser<? extends PineconeOp> apply(ParsedOp op) {
|
||||||
LongFunction<String> spaceFunction = op.getAsFunctionOr("space", "default");
|
LongFunction<String> spaceFunction = op.getAsFunctionOr("space", "default");
|
||||||
LongFunction<PineconeSpace> pcFunction = l -> spaceCache.get(spaceFunction.apply(l));
|
LongFunction<PineconeSpace> pcFunction = l -> spaceCache.get(spaceFunction.apply(l));
|
||||||
|
|
||||||
TypeAndTarget<PineconeOpTypes, String> opType = op.getTypeAndTarget(PineconeOpTypes.class, String.class, "type", "stmt");
|
TypeAndTarget<PineconeOpTypes, String> opType = op.getTypeAndTarget(PineconeOpTypes.class, String.class, "type", "index");
|
||||||
|
|
||||||
LOGGER.info(() -> "Using " + opType.enumId + " statement form for '" + op.getName());
|
LOGGER.info(() -> "Using " + opType.enumId + " statement form for '" + op.getName());
|
||||||
|
|
||||||
|
@ -20,14 +20,17 @@ public class PineconeSpace {
|
|||||||
private final String name;
|
private final String name;
|
||||||
|
|
||||||
private final PineconeClient client;
|
private final PineconeClient client;
|
||||||
private PineconeClientConfig config;
|
private final PineconeClientConfig config;
|
||||||
|
|
||||||
|
private final Map<String,PineconeConnection> connections = new HashMap<String,PineconeConnection>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Connections are index-specific so we need to allow for multiple connection management across indices.
|
* Create a new PineconeSpace Object which stores all stateful contextual information needed to interact
|
||||||
* However, note that a single connection object is thread safe and can be used by multiple clients.
|
* with the Pinecone database instance.
|
||||||
|
*
|
||||||
|
* @param name The name of this space
|
||||||
|
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
|
||||||
*/
|
*/
|
||||||
private Map<String,PineconeConnection> connections = new HashMap<String,PineconeConnection>();
|
|
||||||
|
|
||||||
public PineconeSpace(String name, NBConfiguration cfg) {
|
public PineconeSpace(String name, NBConfiguration cfg) {
|
||||||
this.apiKey = cfg.get("apiKey");
|
this.apiKey = cfg.get("apiKey");
|
||||||
this.environment = cfg.get("environment");
|
this.environment = cfg.get("environment");
|
||||||
@ -42,6 +45,13 @@ public class PineconeSpace {
|
|||||||
this.client = new PineconeClient(config);
|
this.client = new PineconeClient(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connections are index-specific so we need to allow for multiple connection management across indices.
|
||||||
|
* However, note that a single connection object is thread safe and can be used by multiple clients.
|
||||||
|
*
|
||||||
|
* @param index The database index for which a connection is being requested
|
||||||
|
* @return The {@link PineconeConnection} for this database index
|
||||||
|
*/
|
||||||
public synchronized PineconeConnection getConnection(String index) {
|
public synchronized PineconeConnection getConnection(String index) {
|
||||||
PineconeConnection connection = connections.get(index);
|
PineconeConnection connection = connections.get(index);
|
||||||
if (connection == null) {
|
if (connection == null) {
|
||||||
|
@ -5,42 +5,49 @@ import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
|||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeDeleteOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeDeleteOp;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.PineconeConnection;
|
|
||||||
import io.pinecone.proto.DeleteRequest;
|
import io.pinecone.proto.DeleteRequest;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
/**
|
|
||||||
* return DeleteRequest.newBuilder()
|
|
||||||
* .setNamespace(namespace)
|
|
||||||
* .addAllIds(Arrays.asList(idsToDelete))
|
|
||||||
* .setDeleteAll(false)
|
|
||||||
* .build();
|
|
||||||
*/
|
|
||||||
|
|
||||||
public class PineconeDeleteOpDispenser extends PineconeOpDispenser {
|
public class PineconeDeleteOpDispenser extends PineconeOpDispenser {
|
||||||
|
private static final Logger LOGGER = LogManager.getLogger(PineconeDeleteOpDispenser.class);
|
||||||
private final LongFunction<DeleteRequest> deleteRequestFunc;
|
private final LongFunction<DeleteRequest> deleteRequestFunc;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new PineconeDeleteOpDispenser subclassed from {@link PineconeOpDispenser}.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
|
* @param pcFunction A function to return the associated context of this activity (see {@link PineconeSpace})
|
||||||
|
* @param targetFunction A LongFunction that returns the specified Pinecone Index for this Op
|
||||||
|
*/
|
||||||
public PineconeDeleteOpDispenser(PineconeDriverAdapter adapter,
|
public PineconeDeleteOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
LongFunction<PineconeSpace> pcFunction,
|
LongFunction<PineconeSpace> pcFunction,
|
||||||
LongFunction<String> targetFunction) {
|
LongFunction<String> targetFunction) {
|
||||||
super(adapter, op, pcFunction, targetFunction);
|
super(adapter, op, pcFunction, targetFunction);
|
||||||
|
|
||||||
indexNameFunc = op.getAsRequiredFunction("delete", String.class);
|
|
||||||
deleteRequestFunc = createDeleteRequestFunction(op);
|
deleteRequestFunc = createDeleteRequestFunction(op);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PineconeOp apply(long value) {
|
public PineconeOp apply(long value) {
|
||||||
return new PineconeDeleteOp(pcFunction.apply(value).getConnection(indexNameFunc.apply(value)),
|
return new PineconeDeleteOp(pcFunction.apply(value).getConnection(targetFunction.apply(value)),
|
||||||
deleteRequestFunc.apply(value));
|
deleteRequestFunc.apply(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* return DeleteRequest.newBuilder()
|
||||||
|
* .setNamespace(namespace)
|
||||||
|
* .addAllIds(Arrays.asList(idsToDelete))
|
||||||
|
* .setDeleteAll(false)
|
||||||
|
* .build();
|
||||||
|
*/
|
||||||
private LongFunction<DeleteRequest> createDeleteRequestFunction(ParsedOp op) {
|
private LongFunction<DeleteRequest> createDeleteRequestFunction(ParsedOp op) {
|
||||||
LongFunction<DeleteRequest.Builder> rFunc = l -> DeleteRequest.newBuilder();
|
LongFunction<DeleteRequest.Builder> rFunc = l -> DeleteRequest.newBuilder();
|
||||||
|
|
||||||
|
@ -5,31 +5,40 @@ import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
|||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeDescribeIndexStatsOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeDescribeIndexStatsOp;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.PineconeConnection;
|
|
||||||
import io.pinecone.proto.DescribeIndexStatsRequest;
|
import io.pinecone.proto.DescribeIndexStatsRequest;
|
||||||
|
import jakarta.ws.rs.NotSupportedException;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
public class PineconeDescribeIndexStatsOpDispenser extends PineconeOpDispenser {
|
public class PineconeDescribeIndexStatsOpDispenser extends PineconeOpDispenser {
|
||||||
|
private static final Logger LOGGER = LogManager.getLogger(PineconeDescribeIndexStatsOpDispenser.class);
|
||||||
private final LongFunction<DescribeIndexStatsRequest> indexStatsRequestFunc;
|
private final LongFunction<DescribeIndexStatsRequest> indexStatsRequestFunc;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new PineconeDescribeIndexStatsOpDispenser subclassed from {@link PineconeOpDispenser}.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
|
* @param pcFunction A function to return the associated context of this activity (see {@link PineconeSpace})
|
||||||
|
* @param targetFunction A LongFunction that returns the specified Pinecone Index for this Op
|
||||||
|
*/
|
||||||
public PineconeDescribeIndexStatsOpDispenser(PineconeDriverAdapter adapter,
|
public PineconeDescribeIndexStatsOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
LongFunction<PineconeSpace> pcFunction,
|
LongFunction<PineconeSpace> pcFunction,
|
||||||
LongFunction<String> targetFunction) {
|
LongFunction<String> targetFunction) {
|
||||||
super(adapter, op, pcFunction, targetFunction);
|
super(adapter, op, pcFunction, targetFunction);
|
||||||
indexNameFunc = op.getAsRequiredFunction("indexStatsRequest", String.class);
|
|
||||||
indexStatsRequestFunc = createDescribeIndexStatsRequestFunction(op);
|
indexStatsRequestFunc = createDescribeIndexStatsRequestFunction(op);
|
||||||
}
|
}
|
||||||
|
|
||||||
private LongFunction<DescribeIndexStatsRequest> createDescribeIndexStatsRequestFunction(ParsedOp op) {
|
private LongFunction<DescribeIndexStatsRequest> createDescribeIndexStatsRequestFunction(ParsedOp op) {
|
||||||
LongFunction<DescribeIndexStatsRequest.Builder> rFunc = l -> DescribeIndexStatsRequest.newBuilder();
|
throw new NotSupportedException("Pinecone Describe Index Stats Op not yet supported");
|
||||||
return l -> rFunc.apply(l).build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PineconeOp apply(long value) {
|
public PineconeOp apply(long value) {
|
||||||
return new PineconeDescribeIndexStatsOp(pcFunction.apply(value).getConnection(indexNameFunc.apply(value)),
|
return new PineconeDescribeIndexStatsOp(pcFunction.apply(value).getConnection(targetFunction.apply(value)),
|
||||||
indexStatsRequestFunc.apply(value));
|
indexStatsRequestFunc.apply(value));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,8 @@ import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
|||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
import io.pinecone.proto.FetchRequest;
|
import io.pinecone.proto.FetchRequest;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -14,14 +16,22 @@ import java.util.Optional;
|
|||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
public class PineconeFetchOpDispenser extends PineconeOpDispenser {
|
public class PineconeFetchOpDispenser extends PineconeOpDispenser {
|
||||||
|
private static final Logger LOGGER = LogManager.getLogger(PineconeFetchOpDispenser.class);
|
||||||
private final LongFunction<FetchRequest> fetchRequestFunc;
|
private final LongFunction<FetchRequest> fetchRequestFunc;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new PineconeFetchOpDispenser subclassed from {@link PineconeOpDispenser}.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
|
* @param pcFunction A function to return the associated context of this activity (see {@link PineconeSpace})
|
||||||
|
* @param targetFunction A LongFunction that returns the specified Pinecone Index for this Op
|
||||||
|
*/
|
||||||
public PineconeFetchOpDispenser(PineconeDriverAdapter adapter,
|
public PineconeFetchOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
LongFunction<PineconeSpace> pcFunction,
|
LongFunction<PineconeSpace> pcFunction,
|
||||||
LongFunction<String> targetFunction) {
|
LongFunction<String> targetFunction) {
|
||||||
super(adapter, op, pcFunction, targetFunction);
|
super(adapter, op, pcFunction, targetFunction);
|
||||||
indexNameFunc = op.getAsRequiredFunction("fetch", String.class);
|
|
||||||
fetchRequestFunc = createFetchRequestFunction(op);
|
fetchRequestFunc = createFetchRequestFunction(op);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -52,7 +62,7 @@ public class PineconeFetchOpDispenser extends PineconeOpDispenser {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PineconeOp apply(long value) {
|
public PineconeOp apply(long value) {
|
||||||
return new PineconeFetchOp(pcFunction.apply(value).getConnection(indexNameFunc.apply(value)),
|
return new PineconeFetchOp(pcFunction.apply(value).getConnection(targetFunction.apply(value)),
|
||||||
fetchRequestFunc.apply(value));
|
fetchRequestFunc.apply(value));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -26,10 +26,8 @@ import io.pinecone.PineconeConnection;
|
|||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
public abstract class PineconeOpDispenser extends BaseOpDispenser<PineconeOp, PineconeSpace> {
|
public abstract class PineconeOpDispenser extends BaseOpDispenser<PineconeOp, PineconeSpace> {
|
||||||
protected LongFunction<PineconeSpace> pcFunction;
|
protected final LongFunction<PineconeSpace> pcFunction;
|
||||||
protected LongFunction<String> targetFunction;
|
protected final LongFunction<String> targetFunction;
|
||||||
protected LongFunction<String> indexNameFunc;
|
|
||||||
|
|
||||||
|
|
||||||
protected PineconeOpDispenser(PineconeDriverAdapter adapter,
|
protected PineconeOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
|
@ -7,48 +7,56 @@ import io.nosqlbench.adapter.pinecone.ops.PineconeQueryOp;
|
|||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.QueryRequest;
|
import io.pinecone.proto.QueryRequest;
|
||||||
import io.pinecone.proto.QueryVector;
|
import io.pinecone.proto.QueryVector;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
/**
|
|
||||||
float[] rawVector = {1.0F, 2.0F, 3.0F};
|
|
||||||
QueryVector queryVector = QueryVector.newBuilder()
|
|
||||||
.addAllValues(Floats.asList(rawVector))
|
|
||||||
.setFilter(Struct.newBuilder()
|
|
||||||
.putFields("some_field", Value.newBuilder()
|
|
||||||
.setStructValue(Struct.newBuilder()
|
|
||||||
.putFields("$lt", Value.newBuilder()
|
|
||||||
.setNumberValue(3)
|
|
||||||
.build()))
|
|
||||||
.build())
|
|
||||||
.build())
|
|
||||||
.setNamespace("default-namespace")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
QueryRequest queryRequest = QueryRequest.newBuilder()
|
|
||||||
.addQueries(queryVector)
|
|
||||||
.setNamespace("default-namespace")
|
|
||||||
.setTopK(2)
|
|
||||||
.setIncludeMetadata(true)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
**/
|
|
||||||
|
|
||||||
public class PineconeQueryOpDispenser extends PineconeOpDispenser {
|
public class PineconeQueryOpDispenser extends PineconeOpDispenser {
|
||||||
|
private static final Logger LOGGER = LogManager.getLogger(PineconeQueryOpDispenser.class);
|
||||||
private final LongFunction<QueryRequest> queryRequestFunc;
|
private final LongFunction<QueryRequest> queryRequestFunc;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new PineconeQueryOpDispenser subclassed from {@link PineconeOpDispenser}.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
|
* @param pcFunction A function to return the associated context of this activity (see {@link PineconeSpace})
|
||||||
|
* @param targetFunction A LongFunction that returns the specified Pinecone Index for this Op
|
||||||
|
*/
|
||||||
public PineconeQueryOpDispenser(PineconeDriverAdapter adapter,
|
public PineconeQueryOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
LongFunction<PineconeSpace> pcFunction,
|
LongFunction<PineconeSpace> pcFunction,
|
||||||
LongFunction<String> targetFunction) {
|
LongFunction<String> targetFunction) {
|
||||||
super(adapter, op, pcFunction, targetFunction);
|
super(adapter, op, pcFunction, targetFunction);
|
||||||
|
|
||||||
indexNameFunc = op.getAsRequiredFunction("query", String.class);
|
|
||||||
queryRequestFunc = createQueryRequestFunc(op, createQueryVectorFunc(op));
|
queryRequestFunc = createQueryRequestFunc(op, createQueryVectorFunc(op));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
float[] rawVector = {1.0F, 2.0F, 3.0F};
|
||||||
|
QueryVector queryVector = QueryVector.newBuilder()
|
||||||
|
.addAllValues(Floats.asList(rawVector))
|
||||||
|
.setFilter(Struct.newBuilder()
|
||||||
|
.putFields("some_field", Value.newBuilder()
|
||||||
|
.setStructValue(Struct.newBuilder()
|
||||||
|
.putFields("$lt", Value.newBuilder()
|
||||||
|
.setNumberValue(3)
|
||||||
|
.build()))
|
||||||
|
.build())
|
||||||
|
.build())
|
||||||
|
.setNamespace("default-namespace")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
QueryRequest queryRequest = QueryRequest.newBuilder()
|
||||||
|
.addQueries(queryVector)
|
||||||
|
.setNamespace("default-namespace")
|
||||||
|
.setTopK(2)
|
||||||
|
.setIncludeMetadata(true)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
*/
|
||||||
private LongFunction<QueryRequest> createQueryRequestFunc(ParsedOp op, LongFunction<QueryVector> queryVectorFunc) {
|
private LongFunction<QueryRequest> createQueryRequestFunc(ParsedOp op, LongFunction<QueryVector> queryVectorFunc) {
|
||||||
LongFunction<QueryRequest.Builder> rFunc = l -> QueryRequest.newBuilder();
|
LongFunction<QueryRequest.Builder> rFunc = l -> QueryRequest.newBuilder();
|
||||||
|
|
||||||
@ -111,7 +119,7 @@ public class PineconeQueryOpDispenser extends PineconeOpDispenser {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PineconeOp apply(long value) {
|
public PineconeOp apply(long value) {
|
||||||
return new PineconeQueryOp(pcFunction.apply(value).getConnection(indexNameFunc.apply(value)),
|
return new PineconeQueryOp(pcFunction.apply(value).getConnection(targetFunction.apply(value)),
|
||||||
queryRequestFunc.apply(value));
|
queryRequestFunc.apply(value));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5,32 +5,40 @@ import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
|||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeUpdateOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeUpdateOp;
|
||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.PineconeConnection;
|
|
||||||
import io.pinecone.proto.UpdateRequest;
|
import io.pinecone.proto.UpdateRequest;
|
||||||
|
import jakarta.ws.rs.NotSupportedException;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
|
|
||||||
public class PineconeUpdateOpDispenser extends PineconeOpDispenser {
|
public class PineconeUpdateOpDispenser extends PineconeOpDispenser {
|
||||||
|
private static final Logger LOGGER = LogManager.getLogger(PineconeUpdateOpDispenser.class);
|
||||||
private final LongFunction<UpdateRequest> updateRequestFunc;
|
private final LongFunction<UpdateRequest> updateRequestFunc;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new PineconeUpdateOpDispenser subclassed from {@link PineconeOpDispenser}.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
|
* @param pcFunction A function to return the associated context of this activity (see {@link PineconeSpace})
|
||||||
|
* @param targetFunction A LongFunction that returns the specified Pinecone Index for this Op
|
||||||
|
*/
|
||||||
public PineconeUpdateOpDispenser(PineconeDriverAdapter adapter,
|
public PineconeUpdateOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
LongFunction<PineconeSpace> pcFunction,
|
LongFunction<PineconeSpace> pcFunction,
|
||||||
LongFunction<String> targetFunction) {
|
LongFunction<String> targetFunction) {
|
||||||
super(adapter, op, pcFunction, targetFunction);
|
super(adapter, op, pcFunction, targetFunction);
|
||||||
|
|
||||||
indexNameFunc = op.getAsRequiredFunction("update", String.class);
|
|
||||||
updateRequestFunc = createUpdateRequestFunction(op);
|
updateRequestFunc = createUpdateRequestFunction(op);
|
||||||
}
|
}
|
||||||
|
|
||||||
private LongFunction<UpdateRequest> createUpdateRequestFunction(ParsedOp op) {
|
private LongFunction<UpdateRequest> createUpdateRequestFunction(ParsedOp op) {
|
||||||
LongFunction<UpdateRequest.Builder> rFunc = l -> UpdateRequest.newBuilder();
|
throw new NotSupportedException("Pinecone Update Request Op not yet supported");
|
||||||
return l -> rFunc.apply(l).build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PineconeOp apply(long value) {
|
public PineconeOp apply(long value) {
|
||||||
return new PineconeUpdateOp(pcFunction.apply(value).getConnection(indexNameFunc.apply(value)),
|
return new PineconeUpdateOp(pcFunction.apply(value).getConnection(targetFunction.apply(value)),
|
||||||
updateRequestFunc.apply(value));
|
updateRequestFunc.apply(value));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,63 +1,64 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.opdispensers;
|
package io.nosqlbench.adapter.pinecone.opdispensers;
|
||||||
|
|
||||||
import com.google.common.primitives.Floats;
|
|
||||||
import com.google.protobuf.Struct;
|
|
||||||
import com.google.protobuf.Value;
|
|
||||||
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeUpsertOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeUpsertOp;
|
||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.PineconeConnection;
|
|
||||||
import io.pinecone.proto.UpsertRequest;
|
import io.pinecone.proto.UpsertRequest;
|
||||||
import io.pinecone.proto.Vector;
|
import jakarta.ws.rs.NotSupportedException;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
/**
|
|
||||||
* float[][] upsertData = {{1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}, {7.0F, 8.0F, 9.0F}};
|
|
||||||
* List<String> upsertIds = Arrays.asList("v1", "v2", "v3");
|
|
||||||
* List<Vector> upsertVectors = new ArrayList<>();
|
|
||||||
*
|
|
||||||
* for (int i = 0; i < upsertData.length; i++) {
|
|
||||||
* upsertVectors.add(Vector.newBuilder()
|
|
||||||
* .addAllValues(Floats.asList(upsertData[i]))
|
|
||||||
* .setMetadata(Struct.newBuilder()
|
|
||||||
* .putFields("some_field", Value.newBuilder().setNumberValue(i).build())
|
|
||||||
* .build())
|
|
||||||
* .setId(upsertIds.get(i))
|
|
||||||
* .build());
|
|
||||||
* }
|
|
||||||
*
|
|
||||||
* return UpsertRequest.newBuilder()
|
|
||||||
* .addAllVectors(upsertVectors)
|
|
||||||
* .setNamespace("default-namespace")
|
|
||||||
* .build();
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
public class PineconeUpsertOpDispenser extends PineconeOpDispenser {
|
public class PineconeUpsertOpDispenser extends PineconeOpDispenser {
|
||||||
|
private static final Logger LOGGER = LogManager.getLogger(PineconeUpsertOpDispenser.class);
|
||||||
private final LongFunction<UpsertRequest> upsertRequestFunc;
|
private final LongFunction<UpsertRequest> upsertRequestFunc;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new PineconeUpsertOpDispenser subclassed from {@link PineconeOpDispenser}.
|
||||||
|
*
|
||||||
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
|
* @param op The {@link ParsedOp} encapsulating the activity for this cycle
|
||||||
|
* @param pcFunction A function to return the associated context of this activity (see {@link PineconeSpace})
|
||||||
|
* @param targetFunction A LongFunction that returns the specified Pinecone Index for this Op
|
||||||
|
*/
|
||||||
public PineconeUpsertOpDispenser(PineconeDriverAdapter adapter,
|
public PineconeUpsertOpDispenser(PineconeDriverAdapter adapter,
|
||||||
ParsedOp op,
|
ParsedOp op,
|
||||||
LongFunction<PineconeSpace> pcFunction,
|
LongFunction<PineconeSpace> pcFunction,
|
||||||
LongFunction<String> targetFunction) {
|
LongFunction<String> targetFunction) {
|
||||||
super(adapter, op, pcFunction, targetFunction);
|
super(adapter, op, pcFunction, targetFunction);
|
||||||
|
|
||||||
indexNameFunc = op.getAsRequiredFunction("upsert", String.class);
|
|
||||||
upsertRequestFunc = createUpsertRequestFunc(op);
|
upsertRequestFunc = createUpsertRequestFunc(op);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
*float[][] upsertData = {{1.0F, 2.0F, 3.0F}, {4.0F, 5.0F, 6.0F}, {7.0F, 8.0F, 9.0F}};
|
||||||
|
*List<String> upsertIds = Arrays.asList("v1", "v2", "v3");
|
||||||
|
*List<Vector> upsertVectors = new ArrayList<>();
|
||||||
|
*
|
||||||
|
*for (int i = 0; i < upsertData.length; i++) {
|
||||||
|
*upsertVectors.add(Vector.newBuilder()
|
||||||
|
*.addAllValues(Floats.asList(upsertData[i]))
|
||||||
|
*.setMetadata(Struct.newBuilder()
|
||||||
|
*.putFields("some_field", Value.newBuilder().setNumberValue(i).build())
|
||||||
|
*.build())
|
||||||
|
*.setId(upsertIds.get(i))
|
||||||
|
*.build());
|
||||||
|
* }
|
||||||
|
*
|
||||||
|
*return UpsertRequest.newBuilder()
|
||||||
|
*.addAllVectors(upsertVectors)
|
||||||
|
*.setNamespace("default-namespace")
|
||||||
|
*.build();
|
||||||
|
*/
|
||||||
private LongFunction<UpsertRequest> createUpsertRequestFunc(ParsedOp op) {
|
private LongFunction<UpsertRequest> createUpsertRequestFunc(ParsedOp op) {
|
||||||
return null;
|
throw new NotSupportedException("Pinecone Upsert Request Op not yet supported");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public PineconeOp apply(long value) {
|
public PineconeOp apply(long value) {
|
||||||
return new PineconeUpsertOp(pcFunction.apply(value).getConnection(indexNameFunc.apply(value)),
|
return new PineconeUpsertOp(pcFunction.apply(value).getConnection(targetFunction.apply(value)),
|
||||||
upsertRequestFunc.apply(value));
|
upsertRequestFunc.apply(value));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.DeleteRequest;
|
import io.pinecone.proto.DeleteRequest;
|
||||||
import io.pinecone.proto.DeleteResponse;
|
import io.pinecone.proto.DeleteResponse;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
@ -10,8 +11,14 @@ public class PineconeDeleteOp extends PineconeOp {
|
|||||||
|
|
||||||
private static final Logger LOGGER = LogManager.getLogger(PineconeDeleteOp.class);
|
private static final Logger LOGGER = LogManager.getLogger(PineconeDeleteOp.class);
|
||||||
|
|
||||||
private DeleteRequest request;
|
private final DeleteRequest request;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new {@link ParsedOp} encapsulating a call to the Pinecone client delete method
|
||||||
|
*
|
||||||
|
* @param connection The associated {@link PineconeConnection} used to communicate with the database
|
||||||
|
* @param request The {@link DeleteRequest} built for this operation
|
||||||
|
*/
|
||||||
public PineconeDeleteOp(PineconeConnection connection, DeleteRequest request) {
|
public PineconeDeleteOp(PineconeConnection connection, DeleteRequest request) {
|
||||||
super(connection);
|
super(connection);
|
||||||
this.request = request;
|
this.request = request;
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.DescribeIndexStatsRequest;
|
import io.pinecone.proto.DescribeIndexStatsRequest;
|
||||||
import io.pinecone.proto.DescribeIndexStatsResponse;
|
import io.pinecone.proto.DescribeIndexStatsResponse;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
@ -10,8 +11,14 @@ public class PineconeDescribeIndexStatsOp extends PineconeOp {
|
|||||||
|
|
||||||
private static final Logger LOGGER = LogManager.getLogger(PineconeDescribeIndexStatsOp.class);
|
private static final Logger LOGGER = LogManager.getLogger(PineconeDescribeIndexStatsOp.class);
|
||||||
|
|
||||||
private DescribeIndexStatsRequest request;
|
private final DescribeIndexStatsRequest request;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new {@link ParsedOp} encapsulating a call to the Pinecone client describeIndexStats method
|
||||||
|
*
|
||||||
|
* @param connection The associated {@link PineconeConnection} used to communicate with the database
|
||||||
|
* @param request The {@link DescribeIndexStatsRequest} built for this operation
|
||||||
|
*/
|
||||||
public PineconeDescribeIndexStatsOp(PineconeConnection connection, DescribeIndexStatsRequest request) {
|
public PineconeDescribeIndexStatsOp(PineconeConnection connection, DescribeIndexStatsRequest request) {
|
||||||
super(connection);
|
super(connection);
|
||||||
this.request = request;
|
this.request = request;
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.FetchRequest;
|
import io.pinecone.proto.FetchRequest;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
import io.pinecone.proto.FetchResponse;
|
import io.pinecone.proto.FetchResponse;
|
||||||
@ -10,8 +11,14 @@ public class PineconeFetchOp extends PineconeOp {
|
|||||||
|
|
||||||
private static final Logger LOGGER = LogManager.getLogger(PineconeFetchOp.class);
|
private static final Logger LOGGER = LogManager.getLogger(PineconeFetchOp.class);
|
||||||
|
|
||||||
private FetchRequest request;
|
private final FetchRequest request;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new {@link ParsedOp} encapsulating a call to the Pinecone client fetch method
|
||||||
|
*
|
||||||
|
* @param connection The associated {@link PineconeConnection} used to communicate with the database
|
||||||
|
* @param request The {@link FetchRequest} built for this operation
|
||||||
|
*/
|
||||||
public PineconeFetchOp(PineconeConnection connection, FetchRequest request) {
|
public PineconeFetchOp(PineconeConnection connection, FetchRequest request) {
|
||||||
super(connection);
|
super(connection);
|
||||||
this.request = request;
|
this.request = request;
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.QueryRequest;
|
import io.pinecone.proto.QueryRequest;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
import io.pinecone.proto.QueryResponse;
|
import io.pinecone.proto.QueryResponse;
|
||||||
@ -10,8 +11,14 @@ public class PineconeQueryOp extends PineconeOp {
|
|||||||
|
|
||||||
private static final Logger LOGGER = LogManager.getLogger(PineconeQueryOp.class);
|
private static final Logger LOGGER = LogManager.getLogger(PineconeQueryOp.class);
|
||||||
|
|
||||||
private QueryRequest request;
|
private final QueryRequest request;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new {@link ParsedOp} encapsulating a call to the Pinecone client query method
|
||||||
|
*
|
||||||
|
* @param connection The associated {@link PineconeConnection} used to communicate with the database
|
||||||
|
* @param request The {@link QueryRequest} built for this operation
|
||||||
|
*/
|
||||||
public PineconeQueryOp(PineconeConnection connection, QueryRequest request) {
|
public PineconeQueryOp(PineconeConnection connection, QueryRequest request) {
|
||||||
super(connection);
|
super(connection);
|
||||||
this.request = request;
|
this.request = request;
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.UpdateRequest;
|
import io.pinecone.proto.UpdateRequest;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
import io.pinecone.proto.UpdateResponse;
|
import io.pinecone.proto.UpdateResponse;
|
||||||
@ -10,8 +11,14 @@ public class PineconeUpdateOp extends PineconeOp {
|
|||||||
|
|
||||||
private static final Logger LOGGER = LogManager.getLogger(PineconeUpdateOp.class);
|
private static final Logger LOGGER = LogManager.getLogger(PineconeUpdateOp.class);
|
||||||
|
|
||||||
private UpdateRequest request;
|
private final UpdateRequest request;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new {@link ParsedOp} encapsulating a call to the Pinecone client update method
|
||||||
|
*
|
||||||
|
* @param connection The associated {@link PineconeConnection} used to communicate with the database
|
||||||
|
* @param request The {@link UpdateRequest} built for this operation
|
||||||
|
*/
|
||||||
public PineconeUpdateOp(PineconeConnection connection, UpdateRequest request) {
|
public PineconeUpdateOp(PineconeConnection connection, UpdateRequest request) {
|
||||||
super(connection);
|
super(connection);
|
||||||
this.request = request;
|
this.request = request;
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.proto.UpsertRequest;
|
import io.pinecone.proto.UpsertRequest;
|
||||||
import io.pinecone.PineconeConnection;
|
import io.pinecone.PineconeConnection;
|
||||||
import io.pinecone.proto.UpsertResponse;
|
import io.pinecone.proto.UpsertResponse;
|
||||||
@ -10,8 +11,14 @@ public class PineconeUpsertOp extends PineconeOp {
|
|||||||
|
|
||||||
private static final Logger LOGGER = LogManager.getLogger(PineconeUpsertOp.class);
|
private static final Logger LOGGER = LogManager.getLogger(PineconeUpsertOp.class);
|
||||||
|
|
||||||
private UpsertRequest request;
|
private final UpsertRequest request;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new {@link ParsedOp} encapsulating a call to the Pinecone client fetch method
|
||||||
|
*
|
||||||
|
* @param connection The associated {@link PineconeConnection} used to communicate with the database
|
||||||
|
* @param request The {@link UpsertRequest} built for this operation
|
||||||
|
*/
|
||||||
public PineconeUpsertOp(PineconeConnection connection, UpsertRequest request) {
|
public PineconeUpsertOp(PineconeConnection connection, UpsertRequest request) {
|
||||||
super(connection);
|
super(connection);
|
||||||
this.request = request;
|
this.request = request;
|
||||||
|
@ -13,6 +13,17 @@ ops:
|
|||||||
operator:
|
operator:
|
||||||
comparator:
|
comparator:
|
||||||
|
|
||||||
|
# A pinecone query op verbose
|
||||||
|
query-example:
|
||||||
|
type: query
|
||||||
|
index: myindex
|
||||||
|
vector: use bindings to generate an array of floats
|
||||||
|
namespace: mynamespace
|
||||||
|
filter:
|
||||||
|
field:
|
||||||
|
operator:
|
||||||
|
comparator:
|
||||||
|
|
||||||
|
|
||||||
# A delete op
|
# A delete op
|
||||||
delete-example:
|
delete-example:
|
||||||
|
Loading…
Reference in New Issue
Block a user