mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
code cleanup
This commit is contained in:
parent
b0830dc192
commit
0f0784b9a7
@ -40,7 +40,7 @@ public class PineconeOpMapper implements OpMapper<PineconeOp> {
|
|||||||
* Create a new PineconeOpMapper implementing the {@link OpMapper} interface.
|
* Create a new PineconeOpMapper implementing the {@link OpMapper} interface.
|
||||||
*
|
*
|
||||||
* @param adapter The associated {@link PineconeDriverAdapter}
|
* @param adapter The associated {@link PineconeDriverAdapter}
|
||||||
* @param spaceCache A cached context Object of thpe {@link PineconeSpace})
|
* @param spaceCache A cached context Object of the {@link PineconeSpace})
|
||||||
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
|
* @param cfg The configuration ({@link NBConfiguration}) for this nb run
|
||||||
*/
|
*/
|
||||||
public PineconeOpMapper(PineconeDriverAdapter adapter,
|
public PineconeOpMapper(PineconeDriverAdapter adapter,
|
||||||
|
@ -16,9 +16,7 @@
|
|||||||
|
|
||||||
package io.nosqlbench.adapter.pinecone.opdispensers;
|
package io.nosqlbench.adapter.pinecone.opdispensers;
|
||||||
|
|
||||||
import com.google.protobuf.ListValue;
|
|
||||||
import com.google.protobuf.Struct;
|
import com.google.protobuf.Struct;
|
||||||
import com.google.protobuf.Value;
|
|
||||||
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeDeleteOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeDeleteOp;
|
||||||
|
@ -17,7 +17,6 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.opdispensers;
|
package io.nosqlbench.adapter.pinecone.opdispensers;
|
||||||
|
|
||||||
import com.google.protobuf.Struct;
|
import com.google.protobuf.Struct;
|
||||||
import com.google.protobuf.Value;
|
|
||||||
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
import io.nosqlbench.adapter.pinecone.PineconeDriverAdapter;
|
||||||
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
import io.nosqlbench.adapter.pinecone.PineconeSpace;
|
||||||
import io.nosqlbench.adapter.pinecone.ops.PineconeDescribeIndexStatsOp;
|
import io.nosqlbench.adapter.pinecone.ops.PineconeDescribeIndexStatsOp;
|
||||||
@ -54,7 +53,7 @@ public class PineconeDescribeIndexStatsOpDispenser extends PineconeOpDispenser {
|
|||||||
/**
|
/**
|
||||||
* @param op The ParsedOp used to build the Request
|
* @param op The ParsedOp used to build the Request
|
||||||
* @return A function that will take a long (the current cycle) and return a Pinecone DescribeIndexStatsRequest
|
* @return A function that will take a long (the current cycle) and return a Pinecone DescribeIndexStatsRequest
|
||||||
*
|
* <p>
|
||||||
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
||||||
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
||||||
* For each method in the chain a function is created here and added to the chain of functions
|
* For each method in the chain a function is created here and added to the chain of functions
|
||||||
|
@ -53,7 +53,7 @@ public class PineconeFetchOpDispenser extends PineconeOpDispenser {
|
|||||||
/**
|
/**
|
||||||
* @param op The ParsedOp used to build the Request
|
* @param op The ParsedOp used to build the Request
|
||||||
* @return A function that will take a long (the current cycle) and return a Pinecone FetchRequest
|
* @return A function that will take a long (the current cycle) and return a Pinecone FetchRequest
|
||||||
*
|
* <p>
|
||||||
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
||||||
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
||||||
* For each method in the chain a function is created here and added to the chain of functions
|
* For each method in the chain a function is created here and added to the chain of functions
|
||||||
|
@ -25,6 +25,7 @@ import io.nosqlbench.adapter.pinecone.ops.PineconeOp;
|
|||||||
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
|
import io.nosqlbench.engine.api.activityimpl.BaseOpDispenser;
|
||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.function.LongFunction;
|
import java.util.function.LongFunction;
|
||||||
@ -73,6 +74,16 @@ public abstract class PineconeOpDispenser extends BaseOpDispenser<PineconeOp, Pi
|
|||||||
return listValueBuilder.build();
|
return listValueBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected LongFunction<ArrayList<Float>> extractFloatVals(LongFunction<String> af) {
|
||||||
|
return l -> {
|
||||||
|
String[] vals = af.apply(l).split(",");
|
||||||
|
ArrayList<Float> fVals = new ArrayList<>();
|
||||||
|
for (String val : vals) {
|
||||||
|
fVals.add(Float.valueOf(val));
|
||||||
|
}
|
||||||
|
return fVals;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -102,14 +102,8 @@ public class PineconeQueryOpDispenser extends PineconeOpDispenser {
|
|||||||
if (vFunc.isPresent()) {
|
if (vFunc.isPresent()) {
|
||||||
LongFunction<QueryRequest.Builder> finalFunc = rFunc;
|
LongFunction<QueryRequest.Builder> finalFunc = rFunc;
|
||||||
LongFunction<String> af = vFunc.get();
|
LongFunction<String> af = vFunc.get();
|
||||||
LongFunction<ArrayList<Float>> alf = l -> {
|
|
||||||
String[] vals = af.apply(l).split(",");
|
LongFunction<ArrayList<Float>> alf = extractFloatVals(af);
|
||||||
ArrayList<Float> fVals = new ArrayList<>();
|
|
||||||
for (String val : vals) {
|
|
||||||
fVals.add(Float.valueOf(val));
|
|
||||||
}
|
|
||||||
return fVals;
|
|
||||||
};
|
|
||||||
rFunc = l -> finalFunc.apply(l).addAllVector(alf.apply(l));
|
rFunc = l -> finalFunc.apply(l).addAllVector(alf.apply(l));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -123,6 +117,7 @@ public class PineconeQueryOpDispenser extends PineconeOpDispenser {
|
|||||||
return rFunc;
|
return rFunc;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param op the ParsedOp from which the Query Vector objects will be built
|
* @param op the ParsedOp from which the Query Vector objects will be built
|
||||||
* @return an Iterable Collection of QueryVector objects to be added to a Pinecone QueryRequest
|
* @return an Iterable Collection of QueryVector objects to be added to a Pinecone QueryRequest
|
||||||
@ -151,14 +146,12 @@ public class PineconeQueryOpDispenser extends PineconeOpDispenser {
|
|||||||
qvb.setTopK((Integer) vector.get("top_k"));
|
qvb.setTopK((Integer) vector.get("top_k"));
|
||||||
}
|
}
|
||||||
if (vector.containsKey("filter")) {
|
if (vector.containsKey("filter")) {
|
||||||
LongFunction<Struct> builtFilter = buildFilterStruct(l2 -> {
|
LongFunction<Struct> builtFilter = buildFilterStruct(l2 -> (Map) vector.get("filter"));
|
||||||
return (Map) vector.get("filter");
|
|
||||||
});
|
|
||||||
qvb.setFilter(builtFilter.apply(l));
|
qvb.setFilter(builtFilter.apply(l));
|
||||||
}
|
}
|
||||||
if (vector.containsKey("sparse_values")) {
|
if (vector.containsKey("sparse_values")) {
|
||||||
Map<String,String> sparse_values = (Map<String, String>) vector.get("sparse_values");
|
Map<String,String> sparse_values = (Map<String, String>) vector.get("sparse_values");
|
||||||
rawValues = ((String) sparse_values.get("values")).split(",");
|
rawValues = sparse_values.get("values").split(",");
|
||||||
floatValues = new ArrayList<>();
|
floatValues = new ArrayList<>();
|
||||||
for (String val : rawValues) {
|
for (String val : rawValues) {
|
||||||
floatValues.add(Float.valueOf(val));
|
floatValues.add(Float.valueOf(val));
|
||||||
|
@ -59,7 +59,7 @@ public class PineconeUpdateOpDispenser extends PineconeOpDispenser {
|
|||||||
/**
|
/**
|
||||||
* @param op the ParsedOp from which the SparseValues object will be built
|
* @param op the ParsedOp from which the SparseValues object will be built
|
||||||
* @return a SparseValues Object to be added to a Pinecone UpdateRequest
|
* @return a SparseValues Object to be added to a Pinecone UpdateRequest
|
||||||
*
|
* <p>
|
||||||
* This method interrogates the subsection of the ParsedOp defined for SparseValues parameters and constructs
|
* This method interrogates the subsection of the ParsedOp defined for SparseValues parameters and constructs
|
||||||
* a SparseValues Object based on the included values, or returns null if this section is not populated. The
|
* a SparseValues Object based on the included values, or returns null if this section is not populated. The
|
||||||
* base function returns either the SparseValues Object or null, while the interior function builds the SparseValues
|
* base function returns either the SparseValues Object or null, while the interior function builds the SparseValues
|
||||||
@ -88,8 +88,8 @@ public class PineconeUpdateOpDispenser extends PineconeOpDispenser {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @param op the ParsedOp from which the Metadata objects will be built
|
* @param op the ParsedOp from which the Metadata objects will be built
|
||||||
* @return an Metadata Struct to be added to a Pinecone UpdateRequest
|
* @return a Metadata Struct to be added to a Pinecone UpdateRequest
|
||||||
*
|
* <p>
|
||||||
* This method interrogates the subsection of the ParsedOp defined for metadata parameters and constructs
|
* This method interrogates the subsection of the ParsedOp defined for metadata parameters and constructs
|
||||||
* a Metadata Struct based on the included values, or returns null if this section is not populated. The
|
* a Metadata Struct based on the included values, or returns null if this section is not populated. The
|
||||||
* base function returns either the Metadata Struct or null, while the interior function builds the Metadata
|
* base function returns either the Metadata Struct or null, while the interior function builds the Metadata
|
||||||
@ -114,14 +114,14 @@ public class PineconeUpdateOpDispenser extends PineconeOpDispenser {
|
|||||||
/**
|
/**
|
||||||
* @param op The ParsedOp used to build the Request
|
* @param op The ParsedOp used to build the Request
|
||||||
* @return A function that will take a long (the current cycle) and return a Pinecone UpdateRequest Builder
|
* @return A function that will take a long (the current cycle) and return a Pinecone UpdateRequest Builder
|
||||||
*
|
* <p>
|
||||||
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
||||||
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
||||||
* For each method in the chain a function is created here and added to the chain of functions
|
* For each method in the chain a function is created here and added to the chain of functions
|
||||||
* called at time of instantiation.
|
* called at time of instantiation.
|
||||||
*
|
* <p>
|
||||||
* The Metadata and SparseValues objects used by the UpdateRequest are sufficiently sophisticated in their own
|
* The Metadata and SparseValues objects used by the UpdateRequest are sufficiently sophisticated in their own
|
||||||
* building process that they have been broken out into separate methods. At runtime they are built separately
|
* building process that they have been broken out into separate methods. At runtime, they are built separately
|
||||||
* and then added to the build chain by the builder returned by this method.
|
* and then added to the build chain by the builder returned by this method.
|
||||||
*/
|
*/
|
||||||
private LongFunction<UpdateRequest.Builder> createUpdateRequestFunction(ParsedOp op) {
|
private LongFunction<UpdateRequest.Builder> createUpdateRequestFunction(ParsedOp op) {
|
||||||
@ -145,14 +145,7 @@ public class PineconeUpdateOpDispenser extends PineconeOpDispenser {
|
|||||||
if (vFunc.isPresent()) {
|
if (vFunc.isPresent()) {
|
||||||
LongFunction<UpdateRequest.Builder> finalFunc = rFunc;
|
LongFunction<UpdateRequest.Builder> finalFunc = rFunc;
|
||||||
LongFunction<String> af = vFunc.get();
|
LongFunction<String> af = vFunc.get();
|
||||||
LongFunction<ArrayList<Float>> alf = l -> {
|
LongFunction<ArrayList<Float>> alf = extractFloatVals(af);
|
||||||
String[] vals = af.apply(l).split(",");
|
|
||||||
ArrayList<Float> fVals = new ArrayList<>();
|
|
||||||
for (String val : vals) {
|
|
||||||
fVals.add(Float.valueOf(val));
|
|
||||||
}
|
|
||||||
return fVals;
|
|
||||||
};
|
|
||||||
rFunc = l -> finalFunc.apply(l).addAllValues(alf.apply(l));
|
rFunc = l -> finalFunc.apply(l).addAllValues(alf.apply(l));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,7 +58,7 @@ public class PineconeUpsertOpDispenser extends PineconeOpDispenser {
|
|||||||
/**
|
/**
|
||||||
* @param op the ParsedOp from which the Vector objects will be built
|
* @param op the ParsedOp from which the Vector objects will be built
|
||||||
* @return an Iterable Collection of Vector objects to be added to a Pinecone UpsertRequest
|
* @return an Iterable Collection of Vector objects to be added to a Pinecone UpsertRequest
|
||||||
*
|
* <p>
|
||||||
* This method interrogates the subsection of the ParsedOp defined for Vector parameters and constructs
|
* This method interrogates the subsection of the ParsedOp defined for Vector parameters and constructs
|
||||||
* a list of Vectors based on the included values, or returns null if this section is not populated. The
|
* a list of Vectors based on the included values, or returns null if this section is not populated. The
|
||||||
* base function returns either the List of vectors or null, while the interior function builds the vectors
|
* base function returns either the List of vectors or null, while the interior function builds the vectors
|
||||||
@ -82,7 +82,7 @@ public class PineconeUpsertOpDispenser extends PineconeOpDispenser {
|
|||||||
vb.addAllValues(floatValues);
|
vb.addAllValues(floatValues);
|
||||||
if (vector.containsKey("sparse_values")) {
|
if (vector.containsKey("sparse_values")) {
|
||||||
Map<String,String> sparse_values = (Map<String, String>) vector.get("sparse_values");
|
Map<String,String> sparse_values = (Map<String, String>) vector.get("sparse_values");
|
||||||
rawValues = ((String) sparse_values.get("values")).split(",");
|
rawValues = sparse_values.get("values").split(",");
|
||||||
floatValues = new ArrayList<>();
|
floatValues = new ArrayList<>();
|
||||||
for (String val : rawValues) {
|
for (String val : rawValues) {
|
||||||
floatValues.add(Float.valueOf(val));
|
floatValues.add(Float.valueOf(val));
|
||||||
@ -98,7 +98,7 @@ public class PineconeUpsertOpDispenser extends PineconeOpDispenser {
|
|||||||
.build());
|
.build());
|
||||||
}
|
}
|
||||||
if (vector.containsKey("metadata")) {
|
if (vector.containsKey("metadata")) {
|
||||||
Map<String, Value> metadata_map = new HashMap<String, Value>();
|
Map<String, Value> metadata_map = new HashMap<>();
|
||||||
BiConsumer<String,Object> stringToValue = (key, val) -> {
|
BiConsumer<String,Object> stringToValue = (key, val) -> {
|
||||||
Value targetval = null;
|
Value targetval = null;
|
||||||
if (val instanceof String) targetval = Value.newBuilder().setStringValue((String)val).build();
|
if (val instanceof String) targetval = Value.newBuilder().setStringValue((String)val).build();
|
||||||
@ -118,14 +118,14 @@ public class PineconeUpsertOpDispenser extends PineconeOpDispenser {
|
|||||||
/**
|
/**
|
||||||
* @param op The ParsedOp used to build the Request
|
* @param op The ParsedOp used to build the Request
|
||||||
* @return A function that will take a long (the current cycle) and return a Pinecone UpsertRequest Builder
|
* @return A function that will take a long (the current cycle) and return a Pinecone UpsertRequest Builder
|
||||||
*
|
* <p>
|
||||||
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
* The pattern used here is to accommodate the way Request types are constructed for Pinecone.
|
||||||
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
* Requests use a Builder pattern, so at time of instantiation the methods should be chained together.
|
||||||
* For each method in the chain a function is created here and added to the chain of functions
|
* For each method in the chain a function is created here and added to the chain of functions
|
||||||
* called at time of instantiation.
|
* called at time of instantiation.
|
||||||
*
|
* <p>
|
||||||
* The Vector objects used by the UpsertRequest are sufficiently sophisticated in their own
|
* The Vector objects used by the UpsertRequest are sufficiently sophisticated in their own
|
||||||
* building process that they have been broken out into a separate method. At runtime they are built separately
|
* building process that they have been broken out into a separate method. At runtime, they are built separately
|
||||||
* and then added to the build chain by the builder returned by this method.
|
* and then added to the build chain by the builder returned by this method.
|
||||||
*/
|
*/
|
||||||
private LongFunction<UpsertRequest.Builder> createUpsertRequestFunc(ParsedOp op) {
|
private LongFunction<UpsertRequest.Builder> createUpsertRequestFunc(ParsedOp op) {
|
||||||
|
@ -17,10 +17,9 @@
|
|||||||
package io.nosqlbench.adapter.pinecone.ops;
|
package io.nosqlbench.adapter.pinecone.ops;
|
||||||
|
|
||||||
import io.nosqlbench.engine.api.templating.ParsedOp;
|
import io.nosqlbench.engine.api.templating.ParsedOp;
|
||||||
import io.pinecone.PineconeException;
|
import io.pinecone.PineconeConnection;
|
||||||
import io.pinecone.proto.DeleteRequest;
|
import io.pinecone.proto.DeleteRequest;
|
||||||
import io.pinecone.proto.DeleteResponse;
|
import io.pinecone.proto.DeleteResponse;
|
||||||
import io.pinecone.PineconeConnection;
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
@ -44,6 +43,6 @@ public class PineconeDeleteOp extends PineconeOp {
|
|||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
DeleteResponse response = connection.getBlockingStub().delete(request);
|
DeleteResponse response = connection.getBlockingStub().delete(request);
|
||||||
logger.debug("Pincecone delete request successful: " + response.toString());
|
logger.debug("Pinecone delete request successful: " + response.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -43,6 +43,6 @@ public class PineconeUpdateOp extends PineconeOp {
|
|||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
UpdateResponse response = connection.getBlockingStub().update(request);
|
UpdateResponse response = connection.getBlockingStub().update(request);
|
||||||
logger.debug("UpdateResponse succesful: " + response.toString());
|
logger.debug("UpdateResponse successful: " + response.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user