IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint

Browse Source
This commit is contained in:
MarkWolters
2024-05-08 17:58:23 -04:00
parent a76f69d227
commit 0f6772ce3b
13 changed files with 352 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/DataApiSpace.java
View File

@@ -16,7 +16,6 @@
package io.nosqlbench.adapter.dataapi;
import com.datastax.astra.client.Collection;
import com.datastax.astra.client.DataAPIClient;
import com.datastax.astra.client.Database;
import io.nosqlbench.nb.api.config.standard.ConfigModel;
@@ -40,11 +39,13 @@ public class DataApiSpace {
private String astraApiEndpoint;
private DataAPIClient dataAPIClient;
private Database database;
private String namespace;
public DataApiSpace(String name, NBConfiguration cfg) {
this.config = cfg;
this.name = name;
setToken();
setApiEndpoint();
setNamespace();
createClient();
}
@@ -58,17 +59,22 @@ public class DataApiSpace {
private void createClient() {
this.dataAPIClient = new DataAPIClient(astraToken);
this.database = dataAPIClient.getDatabase(astraApiEndpoint);
// database.getCollection("test");
// database.listCollections().forEach(System.out::println);
// Collection collection = database.getCollection("test");
// collection.deleteMany()
if (namespace != null) {
this.database = dataAPIClient.getDatabase(astraApiEndpoint, namespace);
} else {
this.database = dataAPIClient.getDatabase(astraApiEndpoint);
}
}
private void setApiEndpoint() {
this.astraApiEndpoint = config.get("astraApiEndpoint");
}
private void setNamespace() {
Optional<String> maybeNamespace = config.getOptional("namespace");
maybeNamespace.ifPresent(s -> this.namespace = s);
}
private void setToken() {
String tokenFileContents = null;
Optional<String> tokenFilePath = config.getOptional("astraTokenFile");

6
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiCreateCollectionOpDispenser.java
View File

@@ -45,9 +45,9 @@ public class DataApiCreateCollectionOpDispenser extends DataApiOpDispenser {
LongFunction<Integer> af = dimFunc.get();
optionsBldr.vectorDimension(af.apply(l));
}
// COSINE("cosine"),
// EUCLIDEAN("euclidean"),
// DOT_PRODUCT("dot_product");
// COSINE("cosine"),
// EUCLIDEAN("euclidean"),
// DOT_PRODUCT("dot_product");
Optional<LongFunction<String>> simFunc = op.getAsOptionalFunction("similarity", String.class);
if (simFunc.isPresent()) {
LongFunction<String> sf = simFunc.get();

24
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteManyOpDispenser.java
View File

@@ -16,19 +16,41 @@
package io.nosqlbench.adapter.dataapi.opdispensers;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Filter;
import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter;
import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp;
import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteManyOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
public class DataApiDeleteManyOpDispenser extends DataApiOpDispenser {
private static final Logger logger = LogManager.getLogger(DataApiDeleteManyOpDispenser.class);
private final LongFunction<DataApiDeleteManyOp> opFunction;
public DataApiDeleteManyOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = createOpFunction(op);
}
private LongFunction<DataApiDeleteManyOp> createOpFunction(ParsedOp op) {
return (l) -> {
Database db = spaceFunction.apply(l).getDatabase();
Filter filter = getFilterFromOp(op, l);
return new DataApiDeleteManyOp(
db,
db.getCollection(targetFunction.apply(l)),
filter
);
};
}
@Override
public DataApiBaseOp getOp(long value) {
return null;
return opFunction.apply(value);
}
}

40
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiDeleteOneOpDispenser.java
View File

@@ -16,19 +16,57 @@
package io.nosqlbench.adapter.dataapi.opdispensers;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.DeleteOneOptions;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.Filters;
import com.datastax.astra.client.model.Sorts;
import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter;
import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp;
import io.nosqlbench.adapter.dataapi.ops.DataApiDeleteOneOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public class DataApiDeleteOneOpDispenser extends DataApiOpDispenser {
private static final Logger logger = LogManager.getLogger(DataApiDeleteOneOpDispenser.class);
private final LongFunction<DataApiDeleteOneOp> opFunction;
public DataApiDeleteOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = createOpFunction(op);
}
private LongFunction<DataApiDeleteOneOp> createOpFunction(ParsedOp op) {
return (l) -> {
Database db = spaceFunction.apply(l).getDatabase();
Filter filter = getFilterFromOp(op, l);
DeleteOneOptions options = new DeleteOneOptions();
Optional<LongFunction<Map>> sortFunction = op.getAsOptionalFunction("sort", Map.class);
if (sortFunction.isPresent()) {
Map<String,Object> sortFields = sortFunction.get().apply(l);
String sortOrder = sortFields.get("type").toString();
String sortField = sortFields.get("field").toString();
switch(sortOrder) {
case "asc" -> options = options.sort(Sorts.ascending(sortField));
case "desc" -> options = options.sort(Sorts.descending(sortField));
}
}
return new DataApiDeleteOneOp(
db,
db.getCollection(targetFunction.apply(l)),
filter,
options
);
};
}
@Override
public DataApiBaseOp getOp(long value) {
return null;
return opFunction.apply(value);
}
}

40
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneAndDeleteOpDispenser.java
View File

@@ -16,19 +16,57 @@
package io.nosqlbench.adapter.dataapi.opdispensers;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.Filters;
import com.datastax.astra.client.model.FindOneAndDeleteOptions;
import com.datastax.astra.client.model.Sorts;
import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter;
import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp;
import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneAndDeleteOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public class DataApiFindOneAndDeleteOpDispenser extends DataApiOpDispenser {
private static final Logger logger = LogManager.getLogger(DataApiFindOneAndDeleteOpDispenser.class);
private final LongFunction<DataApiFindOneAndDeleteOp> opFunction;
public DataApiFindOneAndDeleteOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = createOpFunction(op);
}
private LongFunction<DataApiFindOneAndDeleteOp> createOpFunction(ParsedOp op) {
return (l) -> {
Database db = spaceFunction.apply(l).getDatabase();
Filter filter = getFilterFromOp(op, l);
FindOneAndDeleteOptions options = new FindOneAndDeleteOptions();
Optional<LongFunction<Map>> sortFunction = op.getAsOptionalFunction("sort", Map.class);
if (sortFunction.isPresent()) {
Map<String,Object> sortFields = sortFunction.get().apply(l);
String sortOrder = sortFields.get("type").toString();
String sortField = sortFields.get("field").toString();
switch(sortOrder) {
case "asc" -> options = options.sort(Sorts.ascending(sortField));
case "desc" -> options = options.sort(Sorts.descending(sortField));
}
}
return new DataApiFindOneAndDeleteOp(
db,
db.getCollection(targetFunction.apply(l)),
filter,
options
);
};
}
@Override
public DataApiBaseOp getOp(long value) {
return null;
return opFunction.apply(value);
}
}

38
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOneOpDispenser.java
View File

@@ -16,19 +16,55 @@
package io.nosqlbench.adapter.dataapi.opdispensers;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.FindOneOptions;
import com.datastax.astra.client.model.Sorts;
import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter;
import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp;
import io.nosqlbench.adapter.dataapi.ops.DataApiFindOneOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public class DataApiFindOneOpDispenser extends DataApiOpDispenser {
private static final Logger logger = LogManager.getLogger(DataApiFindOneOpDispenser.class);
private final LongFunction<DataApiFindOneOp> opFunction;
public DataApiFindOneOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = createOpFunction(op);
}
private LongFunction<DataApiFindOneOp> createOpFunction(ParsedOp op) {
return (l) -> {
Database db = spaceFunction.apply(l).getDatabase();
Filter filter = getFilterFromOp(op, l);
FindOneOptions options = new FindOneOptions();
Optional<LongFunction<Map>> sortFunction = op.getAsOptionalFunction("sort", Map.class);
if (sortFunction.isPresent()) {
Map<String,Object> sortFields = sortFunction.get().apply(l);
String sortOrder = sortFields.get("type").toString();
String sortField = sortFields.get("field").toString();
switch(sortOrder) {
case "asc" -> options = options.sort(Sorts.ascending(sortField));
case "desc" -> options = options.sort(Sorts.descending(sortField));
}
}
return new DataApiFindOneOp(
db,
db.getCollection(targetFunction.apply(l)),
filter,
options
);
};
}
@Override
public DataApiBaseOp getOp(long value) {
return null;
return opFunction.apply(value);
}
}

38
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiFindOpDispenser.java
View File

@@ -16,19 +16,55 @@
package io.nosqlbench.adapter.dataapi.opdispensers;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.FindOptions;
import com.datastax.astra.client.model.Sorts;
import io.nosqlbench.adapter.dataapi.DataApiDriverAdapter;
import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp;
import io.nosqlbench.adapter.dataapi.ops.DataApiFindOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public class DataApiFindOpDispenser extends DataApiOpDispenser {
private static final Logger logger = LogManager.getLogger(DataApiFindOpDispenser.class);
private final LongFunction<DataApiFindOp> opFunction;
public DataApiFindOpDispenser(DataApiDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = createOpFunction(op);
}
private LongFunction<DataApiFindOp> createOpFunction(ParsedOp op) {
return (l) -> {
Database db = spaceFunction.apply(l).getDatabase();
Filter filter = getFilterFromOp(op, l);
FindOptions options = new FindOptions();
Optional<LongFunction<Map>> sortFunction = op.getAsOptionalFunction("sort", Map.class);
if (sortFunction.isPresent()) {
Map<String,Object> sortFields = sortFunction.get().apply(l);
String sortOrder = sortFields.get("type").toString();
String sortField = sortFields.get("field").toString();
switch(sortOrder) {
case "asc" -> options = options.sort(Sorts.ascending(sortField));
case "desc" -> options = options.sort(Sorts.descending(sortField));
}
}
return new DataApiFindOp(
db,
db.getCollection(targetFunction.apply(l)),
filter,
options
);
};
}
@Override
public DataApiBaseOp getOp(long value) {
return null;
return opFunction.apply(value);
}
}

41
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/opdispensers/DataApiOpDispenser.java
View File

@@ -16,12 +16,18 @@
package io.nosqlbench.adapter.dataapi.opdispensers;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.Filters;
import io.nosqlbench.adapter.dataapi.DataApiSpace;
import io.nosqlbench.adapter.dataapi.ops.DataApiBaseOp;
import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public abstract class DataApiOpDispenser extends BaseOpDispenser<DataApiBaseOp, DataApiSpace> {
@@ -35,4 +41,39 @@ public abstract class DataApiOpDispenser extends BaseOpDispenser<DataApiBaseOp,
this.spaceFunction = adapter.getSpaceFunc(op);
}
protected Filter getFilterFromOp(ParsedOp op, long l) {
Filter filter = null;
Optional<LongFunction<Map>> filterFunction = op.getAsOptionalFunction("filters", Map.class);
if (filterFunction.isPresent()) {
Map<String,Object> filters = filterFunction.get().apply(l);
List<Filter> filterList = new ArrayList<>();
String conjunction = "and";
for (Map.Entry<String, Object> entry : filters.entrySet()) {
if (entry.getKey().equalsIgnoreCase("filter")) {
Map<String,Object> filterFields = (Map<String, Object>) entry.getValue();
switch (filterFields.get("operation").toString()) {
case "lt" ->
filterList.add(Filters.lt(filters.get("field").toString(), (long) filters.get("value")));
case "gt" ->
filterList.add(Filters.gt(filters.get("field").toString(), (long) filters.get("value")));
case "eq" -> filterList.add(Filters.eq(filters.get("field").toString(), filters.get("value")));
default -> logger.error("Operation not supported");
}
} else if (entry.getKey().equalsIgnoreCase("conjunction")) {
conjunction = (String) entry.getValue();
} else {
logger.error("Filter " + entry.getKey() + " not supported");
}
}
if (conjunction.equalsIgnoreCase("and")) {
filter = Filters.and(filterList);
} else if (conjunction.equalsIgnoreCase("or")) {
filter = Filters.or(filterList);
} else {
logger.error("Conjunction " + conjunction + " not supported");
}
}
return filter;
}
}

21
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteManyOp.java Normal file
View File

@@ -0,0 +1,21 @@
package io.nosqlbench.adapter.dataapi.ops;
import com.datastax.astra.client.Collection;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Filter;
public class DataApiDeleteManyOp extends DataApiBaseOp {
private final Collection collection;
private final Filter filter;
public DataApiDeleteManyOp(Database db, Collection collection, Filter filter) {
super(db);
this.collection = collection;
this.filter = filter;
}
@Override
public Object apply(long value) {
return collection.deleteMany(filter);
}
}

24
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiDeleteOneOp.java Normal file
View File

@@ -0,0 +1,24 @@
package io.nosqlbench.adapter.dataapi.ops;
import com.datastax.astra.client.Collection;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.DeleteOneOptions;
import com.datastax.astra.client.model.Filter;
public class DataApiDeleteOneOp extends DataApiBaseOp {
private final Collection collection;
private final Filter filter;
private final DeleteOneOptions options;
public DataApiDeleteOneOp(Database db, Collection collection, Filter filter, DeleteOneOptions options) {
super(db);
this.collection = collection;
this.filter = filter;
this.options = options;
}
@Override
public Object apply(long value) {
return collection.deleteOne(filter, options);
}
}

25
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneAndDeleteOp.java Normal file
View File

@@ -0,0 +1,25 @@
package io.nosqlbench.adapter.dataapi.ops;
import com.datastax.astra.client.Collection;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Document;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.FindOneAndDeleteOptions;
public class DataApiFindOneAndDeleteOp extends DataApiBaseOp {
private final Collection<Document> collection;
private final Filter filter;
private final FindOneAndDeleteOptions options;
public DataApiFindOneAndDeleteOp(Database db, Collection<Document> collection, Filter filter, FindOneAndDeleteOptions options) {
super(db);
this.collection = collection;
this.filter = filter;
this.options = options;
}
@Override
public Object apply(long value) {
return collection.findOneAndDelete(filter, options);
}
}

25
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOneOp.java Normal file
View File

@@ -0,0 +1,25 @@
package io.nosqlbench.adapter.dataapi.ops;
import com.datastax.astra.client.Collection;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Document;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.FindOneOptions;
public class DataApiFindOneOp extends DataApiBaseOp {
private final Collection<Document> collection;
private final Filter filter;
private final FindOneOptions options;
public DataApiFindOneOp(Database db, Collection<Document> collection, Filter filter, FindOneOptions options) {
super(db);
this.collection = collection;
this.filter = filter;
this.options = options;
}
@Override
public Object apply(long value) {
return collection.findOne(filter, options);
}
}

26
nb-adapters/adapter-dataapi/src/main/java/io/nosqlbench/adapter/dataapi/ops/DataApiFindOp.java Normal file
View File

@@ -0,0 +1,26 @@
package io.nosqlbench.adapter.dataapi.ops;
import com.datastax.astra.client.Collection;
import com.datastax.astra.client.Database;
import com.datastax.astra.client.model.Document;
import com.datastax.astra.client.model.Filter;
import com.datastax.astra.client.model.FindOneOptions;
import com.datastax.astra.client.model.FindOptions;
public class DataApiFindOp extends DataApiBaseOp {
private final Collection<Document> collection;
private final Filter filter;
private final FindOptions options;
public DataApiFindOp(Database db, Collection<Document> collection, Filter filter, FindOptions options) {
super(db);
this.collection = collection;
this.filter = filter;
this.options = options;
}
@Override
public Object apply(long value) {
return collection.find(filter, options);
}
}