IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

misc fixes: binding function order, CqlVector, cqld4 config loader precedence, CqlVector->NormalizeVector support

Browse Source
This commit is contained in:
Jonathan Shook 2023-06-01 17:28:39 -05:00 committed by jeffbanks
parent fa0afdd388
commit 171ed3d317
9 changed files with 375 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java
View File

@ -80,10 +80,10 @@ public class Cqld4Space implements AutoCloseable {
helpers.applyConfig(cqlHelperCfg);
// add user-provided parameters
NBConfiguration driverCfg = getDriverOptionsModel().extractConfig(cfg);
if (!driverCfg.isEmpty()) {
NBConfiguration nbActivityDriverOptions = getDriverOptionsModel().extractConfig(cfg);
if (!nbActivityDriverOptions.isEmpty()) {
Map<String, Object> remapped = new LinkedHashMap<>();
driverCfg.getMap().forEach((k, v) -> remapped.put(k.substring("driver.".length()), v));
nbActivityDriverOptions.getMap().forEach((k, v) -> remapped.put(k.substring("driver.".length()), v));
Gson gson = new GsonBuilder().setPrettyPrinting().create();
String remappedViaSerdesToSatisfyObtuseConfigAPI = gson.toJson(remapped);
DriverConfigLoader userProvidedOptions = DriverConfigLoader.fromString(remappedViaSerdesToSatisfyObtuseConfigAPI);
@ -94,9 +94,7 @@ public class Cqld4Space implements AutoCloseable {
DriverConfigLoader cfgDefaults = resolveConfigLoader(cfg).orElse(DriverConfigLoader.fromMap(OptionsMap.driverDefaults()));
dcl = new CompositeDriverConfigLoader(dcl, cfgDefaults);
builder.withConfigLoader(dcl);
int port = cfg.getOptional(int.class, "port").orElse(9042);
// int port = cfg.getOptional(int.class, "port").orElse(9042);
Optional<String> scb = cfg.getOptional(String.class, "secureconnectbundle", "scb");
scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream))
@ -107,20 +105,18 @@ public class Cqld4Space implements AutoCloseable {
.map(s -> Arrays.asList(s.split(",")))
.map(
sl -> sl.stream()
.map(n -> new InetSocketAddress(n, port))
.map(n -> new InetSocketAddress(n, cfg.getOptional(int.class, "port").orElse(9042)))
.collect(Collectors.toList())
);
if (scb.isEmpty()) {
if (contactPointsOption.isPresent()) {
builder.addContactPoints(contactPointsOption.get());
Optional<String> localdc = cfg.getOptional("localdc");
builder.withLocalDatacenter(localdc.orElseThrow(
() -> new BasicError("Starting with driver 4.0, you must specify the local datacenter name with any specified contact points. Example: (use caution) localdc=datacenter1")
));
} else {
builder.addContactPoints(List.of(new InetSocketAddress("localhost", port)));
}
if (contactPointsOption.isPresent()) {
builder.addContactPoints(contactPointsOption.get());
Optional<String> localdc = cfg.getOptional("localdc");
builder.withLocalDatacenter(localdc.orElseThrow(
() -> new BasicError("Starting with driver 4.0, you must specify the local datacenter name with any specified contact points. Example: (use caution) localdc=datacenter1")
));
} else {
builder.addContactPoints(List.of(new InetSocketAddress("localhost", cfg.getOptional(int.class, "port").orElse(9042))));
}
// builder.withCompression(ProtocolOptions.Compression.NONE);
@ -203,6 +199,7 @@ public class Cqld4Space implements AutoCloseable {
builder.withSslContext(ctx);
}
builder.withConfigLoader(dcl);
CqlSession session = builder.build();
return session;
}
@ -244,9 +241,9 @@ public class Cqld4Space implements AutoCloseable {
// URLs
try {
Optional<Content<?>> removeconf = NBIO.remote().pathname(driverconfig).first();
if (removeconf.isPresent()) {
loaders.add(DriverConfigLoader.fromUrl(removeconf.get().getURI().toURL()));
Optional<Content<?>> remoteconf = NBIO.remote().pathname(driverconfig).first();
if (remoteconf.isPresent()) {
loaders.add(DriverConfigLoader.fromUrl(remoteconf.get().getURI().toURL()));
continue;
}
} catch (Exception e) {
@ -284,7 +281,7 @@ public class Cqld4Space implements AutoCloseable {
.add(Param.optional("localdc"))
.add(Param.optional(List.of("secureconnectbundle", "scb")))
.add(Param.optional(List.of("hosts", "host")))
.add(Param.defaultTo("port",9042))
.add(Param.defaultTo("port", 9042))
.add(Param.optional("driverconfig", String.class))
.add(Param.optional("username", String.class, "user name (see also password and passfile)"))
.add(Param.optional("userfile", String.class, "file to load the username from"))

61
adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/geometry/NormalizeVector.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.datamappers.functions.geometry;
import com.datastax.oss.driver.api.core.data.CqlVector;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_vector.NormalizeDoubleVectorList;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_vector.NormalizeFloatVectorList;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
/**
* Normalize a vector in List<Number> form, calling the appropriate conversion function
* depending on the component (Class) type of the incoming List values.
*/
@ThreadSafeMapper
@Categories(Category.experimental)
public class NormalizeVector implements Function<com.datastax.oss.driver.api.core.data.CqlVector ,List> {
private final NormalizeDoubleVectorList ndv = new NormalizeDoubleVectorList();
private final NormalizeFloatVectorList nfv = new NormalizeFloatVectorList();
@Override
public List apply(CqlVector cqlVector) {
Iterable values = cqlVector.getValues();
List<Object> list = new ArrayList<>();
values.forEach(list::add);
if (list.size()==0) {
return List.of();
} else if (list.get(0) instanceof Float) {
List<Float> floats = new ArrayList<>();
list.forEach(o -> floats.add((Float)o));
return nfv.apply(floats);
} else if (list.get(0) instanceof Double) {
List<Double> doubles = new ArrayList<>();
list.forEach(o -> doubles.add((Double) o));
return ndv.apply(doubles);
} else {
throw new RuntimeException("Only Doubles and Floats are recognized.");
}
}
}

18
adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/to_cqlvector/CqlVector.java
View File

@ -21,15 +21,13 @@ import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection.ListSizedHashed;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_float.HashRange;
import java.util.List;
import java.util.function.LongFunction;
/**
* Create a new CqlVector from a composed function, where the inner function
* is a list generation function.
* is a list generation function that must take a long (cycle) input.
*/
@ThreadSafeMapper
@Categories(Category.HOF)
@ -41,15 +39,15 @@ public class CqlVector implements LongFunction<com.datastax.oss.driver.api.core.
public CqlVector(Object func) {
this.func = VirtDataConversions.adaptFunction(func,LongFunction.class, List.class);
}
@Example({"CqlVector()","Create a default 5-component vector with unit-interval components."})
public CqlVector() {
this(new ListSizedHashed(5, new HashRange(0.0f, 1.0f)));
}
//
// @Example({"CqlVector()","Create a default 5-component vector with unit-interval components."})
// public CqlVector() {
// this(new ListSizedHashed(5, new HashRange(0.0f, 1.0f)));
// }
@Override
public com.datastax.oss.driver.api.core.data.CqlVector apply(long value) {
List components = func.apply(value);
public com.datastax.oss.driver.api.core.data.CqlVector apply(long cycle) {
List components = func.apply(cycle);
com.datastax.oss.driver.api.core.data.CqlVector.Builder vbuilder = com.datastax.oss.driver.api.core.data.CqlVector.builder();
vbuilder.add(components.toArray());
return vbuilder.build();

14
adapter-cqld4/src/main/resources/activities/examples/bindings/cqlvectors.yaml
View File

@ -8,13 +8,13 @@ scenarios:
hof-tenunit: run driver=stdout bindings='"'hof_ten.*' cycles=10 format=readout
bindings:
# # default provides a 5-component vector, with unit-interval values. (Not normalized)
# simple_vector: CqlVector()
# # create a HOF CqlVector(dim 4) binding which composes around a long -> list function
# hof_vector: CqlVector(ListSizedHashed(4,HashRange(0.0f,1.0f)))
# # create a HOF CqlVector binding
# hof_vary_vector: CqlVector(ListSizedHashed(HashRange(3,5)->int,HashRange(0.0f,1.0f)))
# default provides a 5-component vector, with unit-interval values. (Not normalized)
simple_vector: CqlVector()
# create a HOF CqlVector(dim 4) binding which composes around a long -> list function
hof_vector: CqlVector(ListSizedHashed(4,HashRange(0.0f,1.0f)))
# create a HOF CqlVector binding
hof_vary_vector: CqlVector(ListSizedHashed(HashRange(3,5)->int,HashRange(0.0f,1.0f)))
# create a normalized vectors of dimension 10
hof_ten_unit: CqlVector(NormalizeVector(ListSizedHashed(10,HashRange(0.0f,1.0f))))
hof_ten_unit: MyCqlVector(ListSizedHashed(10,HashRange(0.0f,1.0f))); NormalizeVector();

117
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/ArgsComparator.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.core.bindings;
import org.apache.commons.lang3.ClassUtils;
import java.lang.reflect.Constructor;
import java.util.*;
public class ArgsComparator implements Comparator<Constructor<?>> {
public enum MATCHRANK {
DIRECT,
CONVERTED,
BOXED,
INCOMPATIBLE
}
private final Object[] parameters;
public ArgsComparator(Object[] parameters) {
this.parameters = parameters;
}
private static final Map<Class<?>, Class<?>> WRAPPER_TYPE_MAP = new HashMap<>(32) {{
put(Integer.class, int.class);
put(Byte.class, byte.class);
put(Character.class, char.class);
put(Boolean.class, boolean.class);
put(Double.class, double.class);
put(Float.class, float.class);
put(Long.class, long.class);
put(Short.class, short.class);
put(Void.class, void.class);
put(int.class, Integer.class);
put(byte.class, Byte.class);
put(char.class, Character.class);
put(boolean.class, Boolean.class);
put(double.class, Double.class);
put(float.class, Float.class);
put(long.class, Long.class);
put(short.class, Short.class);
put(void.class, Void.class);
}};
@Override
public int compare(Constructor<?> o1, Constructor<?> o2) {
return Integer.compare(matchRank(o1, parameters).ordinal(), matchRank(o2, parameters).ordinal());
}
/**
* Establish a priority value (lower is better) based on how well the arguments
* match to the given constructor's parameters.
*
* Note: The distinction between primitives and boxed types is lost here,
* as the primitive version of Class<?> is only accessible via {@link Long#TYPE}
* and similar, so primitive matching and auto-boxed matching are effectively
* the same rank.
*
* rank 0 -> all arguments are the same type or boxed type
* rank 1 -> all arguments are assignable, without autoboxing
* rank 2 -> all arguments are assignable, with autoboxing
* rank 3 -> not all arguments are assignable
* @param ctor - constructor
* @param arguments - arguments to match against
* @return a lower number for when arguments match parameters better
*/
public MATCHRANK matchRank(Constructor<?> ctor, Object[] arguments) {
int paramLen = ctor.getParameterCount();
int argsLen = arguments.length;
if (paramLen!=argsLen && !ctor.isVarArgs()) {
return MATCHRANK.INCOMPATIBLE;
}
int len = arguments.length; // only consider varargs if some provided
MATCHRANK[] ranks = new MATCHRANK[len];
Class<?>[] ptypes = ctor.getParameterTypes();
Class<?>[] atypes = Arrays.stream(arguments).map(Object::getClass).toArray(i -> new Class<?>[i]);
for (int position = 0; position < len; position++) {
Class<?> ptype = ptypes[position];
Class<?> atype = (position<atypes.length) ? atypes[position] : atypes[atypes.length-1];
Class<?> across = WRAPPER_TYPE_MAP.get(atype);
Class<?> pcross = WRAPPER_TYPE_MAP.get(ptype);
if (atype.isPrimitive()==ptype.isPrimitive() && atype.equals(ptype)) {
ranks[position] = MATCHRANK.DIRECT;
} else if (across != null && pcross != null && (across.equals(ptype) || pcross.equals(atype))) {
ranks[position] = MATCHRANK.DIRECT;
} else if (ClassUtils.isAssignable(atype, ptype, false)) {
ranks[position] = MATCHRANK.CONVERTED;
} else if (ClassUtils.isAssignable(atype, ptype, true)) {
ranks[position] = MATCHRANK.BOXED;
} else {
ranks[position] = MATCHRANK.INCOMPATIBLE;
}
}
Integer maxOrdinal = Arrays.stream(ranks).map(MATCHRANK::ordinal).max(Integer::compare).orElse(0);
return MATCHRANK.values()[maxOrdinal];
}
}

4
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -115,7 +115,7 @@ public class VirtDataComposer {
Object[] fargs = fcall.getArguments();
Object[][] params = new Object[fargs.length][];
for (int pos = 0; pos <fargs.length; pos++) {
for (int pos = 0; pos <fargs.length; pos++) { // Resolve functions recursively if needed
Object param = fargs[pos];
if (param instanceof FunctionCall) {

4
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataFunctionResolver.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -97,6 +97,8 @@ public class VirtDataFunctionResolver {
// })
.collect(Collectors.toList());
Collections.sort(matchingConstructors,new ArgsComparator(parameters));
if (returnType != null && inputType != null && matchingConstructors.size() > 1) {
throw new RuntimeException(
"found more than one (" + matchingConstructors.size() + ") matching constructor for " +

86
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/bindings/ArgsComparatorTest.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.core.bindings;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Constructor;
import static org.assertj.core.api.Assertions.assertThat;
public class ArgsComparatorTest {
static Constructor<?> ctor_longBoxed = LongBoxed.class.getConstructors()[0];
static Constructor<?> ctor_longs = Longs.class.getConstructors()[0];
static Constructor<?> ctor_mixed = Mixed.class.getConstructors()[0];
static Constructor<?> ctor_boxed = Boxed.class.getConstructors()[0];
static Constructor<?> ctor_primitives = Primitives.class.getConstructors()[0];
@Test
public void verifyRanks() {
Object[] args = {1, 2};
ArgsComparator comparator = new ArgsComparator(args);
assertThat(comparator.matchRank(ctor_longBoxed,args)).isEqualTo(ArgsComparator.MATCHRANK.INCOMPATIBLE);
assertThat(comparator.matchRank(ctor_longs,args)).isEqualTo(ArgsComparator.MATCHRANK.BOXED);
assertThat(comparator.matchRank(ctor_mixed,args)).isEqualTo(ArgsComparator.MATCHRANK.DIRECT);
assertThat(comparator.matchRank(ctor_boxed,args)).isEqualTo(ArgsComparator.MATCHRANK.DIRECT);
assertThat(comparator.matchRank(ctor_primitives,args)).isEqualTo(ArgsComparator.MATCHRANK.DIRECT);
}
@Test
public void testCtorSanity() {
Object[] args = {1, 2};
try {
ctor_boxed.newInstance(args);
ctor_longs.newInstance(args);
ctor_mixed.newInstance(args);
ctor_primitives.newInstance(args);
// ctor_longBoxed.newInstance(args);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static class Primitives {
public Primitives(int a, int b) {
}
}
private static class Boxed {
public Boxed(Integer a, Integer b) {
}
}
private static class Mixed {
public Mixed(int a, Integer b) {
}
}
private static class Longs {
public Longs(long a, long b) {
}
}
private static class LongBoxed {
public LongBoxed(Long a, Long b) {}
}
}

73
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/bindings/VirtDataFunctionResolverTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.core.bindings;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.Collections;
import static org.assertj.core.api.Assertions.assertThat;
public class VirtDataFunctionResolverTest {
@Test
public void testArgSorting() {
ArgsComparator comparator =
new ArgsComparator(new Object[]{1, 2});
ArrayList<Constructor<?>> ctors = new ArrayList<>();
Constructor<?> long_boxed = LongBoxed.class.getConstructors()[0];
Constructor<?> longs = Longs.class.getConstructors()[0];
Constructor<?> mixed = Mixed.class.getConstructors()[0];
Constructor<?> boxed = Boxed.class.getConstructors()[0];
Constructor<?> primitives = Primitives.class.getConstructors()[0];
ctors.add(long_boxed);
ctors.add(longs);
ctors.add(mixed);
ctors.add(boxed);
ctors.add(primitives);
Collections.sort(ctors,comparator);
assertThat(ctors).containsExactly(mixed, boxed, primitives,longs,long_boxed);
}
private static class Primitives {
public Primitives(int a, int b) {
}
}
private static class Boxed {
public Boxed(Integer a, Integer b) {
}
}
private static class Mixed {
public Mixed(int a, Integer b) {
}
}
private static class Longs {
public Longs(long a, long b) {
}
}
private static class LongBoxed {
public LongBoxed(Long a, Long b) {}
}
}