misc fixes: binding function order, CqlVector, cqld4 config loader precedence, CqlVector->NormalizeVector support

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java

@@ -80,10 +80,10 @@ public class Cqld4Space implements AutoCloseable {
         helpers.applyConfig(cqlHelperCfg);
 
         // add user-provided parameters
-        NBConfiguration driverCfg = getDriverOptionsModel().extractConfig(cfg);
+        NBConfiguration nbActivityDriverOptions = getDriverOptionsModel().extractConfig(cfg);
-        if (!driverCfg.isEmpty()) {
+        if (!nbActivityDriverOptions.isEmpty()) {
             Map<String, Object> remapped = new LinkedHashMap<>();
-            driverCfg.getMap().forEach((k, v) -> remapped.put(k.substring("driver.".length()), v));
+            nbActivityDriverOptions.getMap().forEach((k, v) -> remapped.put(k.substring("driver.".length()), v));
             Gson gson = new GsonBuilder().setPrettyPrinting().create();
             String remappedViaSerdesToSatisfyObtuseConfigAPI = gson.toJson(remapped);
             DriverConfigLoader userProvidedOptions = DriverConfigLoader.fromString(remappedViaSerdesToSatisfyObtuseConfigAPI);
@@ -94,9 +94,7 @@ public class Cqld4Space implements AutoCloseable {
         DriverConfigLoader cfgDefaults = resolveConfigLoader(cfg).orElse(DriverConfigLoader.fromMap(OptionsMap.driverDefaults()));
         dcl = new CompositeDriverConfigLoader(dcl, cfgDefaults);
 
-        builder.withConfigLoader(dcl);
+//        int port = cfg.getOptional(int.class, "port").orElse(9042);
-
-        int port = cfg.getOptional(int.class, "port").orElse(9042);
 
         Optional<String> scb = cfg.getOptional(String.class, "secureconnectbundle", "scb");
         scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream))
@@ -107,11 +105,10 @@ public class Cqld4Space implements AutoCloseable {
             .map(s -> Arrays.asList(s.split(",")))
             .map(
                 sl -> sl.stream()
-                    .map(n -> new InetSocketAddress(n, port))
+                    .map(n -> new InetSocketAddress(n, cfg.getOptional(int.class, "port").orElse(9042)))
                     .collect(Collectors.toList())
             );
 
-        if (scb.isEmpty()) {
         if (contactPointsOption.isPresent()) {
             builder.addContactPoints(contactPointsOption.get());
             Optional<String> localdc = cfg.getOptional("localdc");
@@ -119,8 +116,7 @@ public class Cqld4Space implements AutoCloseable {
                 () -> new BasicError("Starting with driver 4.0, you must specify the local datacenter name with any specified contact points. Example: (use caution) localdc=datacenter1")
             ));
         } else {
-                builder.addContactPoints(List.of(new InetSocketAddress("localhost", port)));
+            builder.addContactPoints(List.of(new InetSocketAddress("localhost", cfg.getOptional(int.class, "port").orElse(9042))));
-            }
         }
 
 //        builder.withCompression(ProtocolOptions.Compression.NONE);
@@ -203,6 +199,7 @@ public class Cqld4Space implements AutoCloseable {
             builder.withSslContext(ctx);
         }
 
+        builder.withConfigLoader(dcl);
         CqlSession session = builder.build();
         return session;
     }
@@ -244,9 +241,9 @@ public class Cqld4Space implements AutoCloseable {
 
             // URLs
             try {
-                Optional<Content<?>> removeconf = NBIO.remote().pathname(driverconfig).first();
+                Optional<Content<?>> remoteconf = NBIO.remote().pathname(driverconfig).first();
-                if (removeconf.isPresent()) {
+                if (remoteconf.isPresent()) {
-                    loaders.add(DriverConfigLoader.fromUrl(removeconf.get().getURI().toURL()));
+                    loaders.add(DriverConfigLoader.fromUrl(remoteconf.get().getURI().toURL()));
                     continue;
                 }
             } catch (Exception e) {
@@ -284,7 +281,7 @@ public class Cqld4Space implements AutoCloseable {
             .add(Param.optional("localdc"))
             .add(Param.optional(List.of("secureconnectbundle", "scb")))
             .add(Param.optional(List.of("hosts", "host")))
-            .add(Param.defaultTo("port",9042))
+            .add(Param.defaultTo("port", 9042))
             .add(Param.optional("driverconfig", String.class))
             .add(Param.optional("username", String.class, "user name (see also password and passfile)"))
             .add(Param.optional("userfile", String.class, "file to load the username from"))

adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/geometry/NormalizeVector.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.datamappers.functions.geometry;
+
+import com.datastax.oss.driver.api.core.data.CqlVector;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.basics.shared.from_long.to_vector.NormalizeDoubleVectorList;
+import io.nosqlbench.virtdata.library.basics.shared.from_long.to_vector.NormalizeFloatVectorList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * Normalize a vector in List<Number> form, calling the appropriate conversion function
+ * depending on the component (Class) type of the incoming List values.
+ */
+@ThreadSafeMapper
+@Categories(Category.experimental)
+public class NormalizeVector implements Function<com.datastax.oss.driver.api.core.data.CqlVector ,List> {
+    private final NormalizeDoubleVectorList ndv = new NormalizeDoubleVectorList();
+    private final NormalizeFloatVectorList nfv = new NormalizeFloatVectorList();
+
+
+    @Override
+    public List apply(CqlVector cqlVector) {
+        Iterable values = cqlVector.getValues();
+        List<Object> list = new ArrayList<>();
+        values.forEach(list::add);
+
+        if (list.size()==0) {
+            return List.of();
+        } else if (list.get(0) instanceof Float) {
+            List<Float> floats = new ArrayList<>();
+            list.forEach(o -> floats.add((Float)o));
+            return nfv.apply(floats);
+        } else if (list.get(0) instanceof Double) {
+            List<Double> doubles = new ArrayList<>();
+            list.forEach(o -> doubles.add((Double) o));
+            return ndv.apply(doubles);
+        } else {
+            throw new RuntimeException("Only Doubles and Floats are recognized.");
+        }
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/to_cqlvector/CqlVector.java

@@ -21,15 +21,13 @@ import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.Example;
 import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
 import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
-import io.nosqlbench.virtdata.library.basics.shared.from_long.to_collection.ListSizedHashed;
-import io.nosqlbench.virtdata.library.basics.shared.from_long.to_float.HashRange;
 
 import java.util.List;
 import java.util.function.LongFunction;
 
 /**
  * Create a new CqlVector from a composed function, where the inner function
- * is a list generation function.
+ * is a list generation function that must take a long (cycle) input.
  */
 @ThreadSafeMapper
 @Categories(Category.HOF)
@@ -41,15 +39,15 @@ public class CqlVector implements LongFunction<com.datastax.oss.driver.api.core.
     public CqlVector(Object func) {
         this.func = VirtDataConversions.adaptFunction(func,LongFunction.class, List.class);
     }
-
+//
-    @Example({"CqlVector()","Create a default 5-component vector with unit-interval components."})
+//    @Example({"CqlVector()","Create a default 5-component vector with unit-interval components."})
-    public CqlVector() {
+//    public CqlVector() {
-        this(new ListSizedHashed(5, new HashRange(0.0f, 1.0f)));
+//        this(new ListSizedHashed(5, new HashRange(0.0f, 1.0f)));
-    }
+//    }
 
     @Override
-    public com.datastax.oss.driver.api.core.data.CqlVector apply(long value) {
+    public com.datastax.oss.driver.api.core.data.CqlVector apply(long cycle) {
-        List components = func.apply(value);
+        List components = func.apply(cycle);
         com.datastax.oss.driver.api.core.data.CqlVector.Builder vbuilder = com.datastax.oss.driver.api.core.data.CqlVector.builder();
         vbuilder.add(components.toArray());
         return vbuilder.build();

adapter-cqld4/src/main/resources/activities/examples/bindings/cqlvectors.yaml

@@ -8,13 +8,13 @@ scenarios:
   hof-tenunit: run driver=stdout bindings='"'hof_ten.*' cycles=10 format=readout
 
 bindings:
-#  # default provides a 5-component vector, with unit-interval values. (Not normalized)
+  # default provides a 5-component vector, with unit-interval values. (Not normalized)
-#  simple_vector: CqlVector()
+  simple_vector: CqlVector()
-#  # create a HOF CqlVector(dim 4) binding which composes around a long -> list function
+  # create a HOF CqlVector(dim 4) binding which composes around a long -> list function
-#  hof_vector: CqlVector(ListSizedHashed(4,HashRange(0.0f,1.0f)))
+  hof_vector: CqlVector(ListSizedHashed(4,HashRange(0.0f,1.0f)))
-#  # create a HOF CqlVector binding
+  # create a HOF CqlVector binding
-#  hof_vary_vector: CqlVector(ListSizedHashed(HashRange(3,5)->int,HashRange(0.0f,1.0f)))
+  hof_vary_vector: CqlVector(ListSizedHashed(HashRange(3,5)->int,HashRange(0.0f,1.0f)))
   # create a normalized vectors of dimension 10
-  hof_ten_unit: CqlVector(NormalizeVector(ListSizedHashed(10,HashRange(0.0f,1.0f))))
+  hof_ten_unit: MyCqlVector(ListSizedHashed(10,HashRange(0.0f,1.0f))); NormalizeVector();
 
 

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/ArgsComparator.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.core.bindings;
+
+import org.apache.commons.lang3.ClassUtils;
+
+import java.lang.reflect.Constructor;
+import java.util.*;
+
+public class ArgsComparator implements Comparator<Constructor<?>> {
+    public enum MATCHRANK {
+        DIRECT,
+        CONVERTED,
+        BOXED,
+        INCOMPATIBLE
+
+    }
+    private final Object[] parameters;
+
+    public ArgsComparator(Object[] parameters) {
+        this.parameters = parameters;
+    }
+
+    private static final Map<Class<?>, Class<?>> WRAPPER_TYPE_MAP = new HashMap<>(32) {{
+        put(Integer.class, int.class);
+        put(Byte.class, byte.class);
+        put(Character.class, char.class);
+        put(Boolean.class, boolean.class);
+        put(Double.class, double.class);
+        put(Float.class, float.class);
+        put(Long.class, long.class);
+        put(Short.class, short.class);
+        put(Void.class, void.class);
+
+        put(int.class, Integer.class);
+        put(byte.class, Byte.class);
+        put(char.class, Character.class);
+        put(boolean.class, Boolean.class);
+        put(double.class, Double.class);
+        put(float.class, Float.class);
+        put(long.class, Long.class);
+        put(short.class, Short.class);
+        put(void.class, Void.class);
+    }};
+
+    @Override
+    public int compare(Constructor<?> o1, Constructor<?> o2) {
+        return Integer.compare(matchRank(o1, parameters).ordinal(), matchRank(o2, parameters).ordinal());
+    }
+
+    /**
+     * Establish a priority value (lower is better) based on how well the arguments
+     * match to the given constructor's parameters.
+     *
+     * Note: The distinction between primitives and boxed types is lost here,
+     * as the primitive version of Class<?> is only accessible via {@link Long#TYPE}
+     * and similar, so primitive matching and auto-boxed matching are effectively
+     * the same rank.
+     *
+     *  rank 0 -> all arguments are the same type or boxed type
+     *  rank 1 -> all arguments are assignable, without autoboxing
+     *  rank 2 -> all arguments are assignable, with autoboxing
+     *  rank 3 -> not all arguments are assignable
+     * @param ctor - constructor
+     * @param arguments - arguments to match against
+     * @return a lower number for when arguments match parameters better
+     */
+    public MATCHRANK matchRank(Constructor<?> ctor, Object[] arguments) {
+        int paramLen = ctor.getParameterCount();
+        int argsLen = arguments.length;
+
+        if (paramLen!=argsLen && !ctor.isVarArgs()) {
+            return MATCHRANK.INCOMPATIBLE;
+        }
+
+        int len = arguments.length; // only consider varargs if some provided
+        MATCHRANK[] ranks = new MATCHRANK[len];
+
+        Class<?>[] ptypes = ctor.getParameterTypes();
+        Class<?>[] atypes = Arrays.stream(arguments).map(Object::getClass).toArray(i -> new Class<?>[i]);
+
+        for (int position = 0; position < len; position++) {
+            Class<?> ptype = ptypes[position];
+            Class<?> atype = (position<atypes.length) ? atypes[position] : atypes[atypes.length-1];
+            Class<?> across = WRAPPER_TYPE_MAP.get(atype);
+            Class<?> pcross = WRAPPER_TYPE_MAP.get(ptype);
+
+            if (atype.isPrimitive()==ptype.isPrimitive() && atype.equals(ptype)) {
+                ranks[position] = MATCHRANK.DIRECT;
+            } else if (across != null && pcross != null && (across.equals(ptype) || pcross.equals(atype))) {
+                ranks[position] = MATCHRANK.DIRECT;
+            } else if (ClassUtils.isAssignable(atype, ptype, false)) {
+                ranks[position] = MATCHRANK.CONVERTED;
+            } else if (ClassUtils.isAssignable(atype, ptype, true)) {
+                ranks[position] = MATCHRANK.BOXED;
+            } else {
+                ranks[position] = MATCHRANK.INCOMPATIBLE;
+            }
+        }
+        Integer maxOrdinal = Arrays.stream(ranks).map(MATCHRANK::ordinal).max(Integer::compare).orElse(0);
+        return MATCHRANK.values()[maxOrdinal];
+    }
+}

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -115,7 +115,7 @@ public class VirtDataComposer {
         Object[] fargs = fcall.getArguments();
         Object[][] params = new Object[fargs.length][];
 
-        for (int pos = 0; pos <fargs.length; pos++) {
+        for (int pos = 0; pos <fargs.length; pos++) { // Resolve functions recursively if needed
             Object param = fargs[pos];
             if (param instanceof FunctionCall) {
 

virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataFunctionResolver.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 nosqlbench
+ * Copyright (c) 2022-2023 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -97,6 +97,8 @@ public class VirtDataFunctionResolver {
 //                })
                 .collect(Collectors.toList());
 
+        Collections.sort(matchingConstructors,new ArgsComparator(parameters));
+
         if (returnType != null && inputType != null && matchingConstructors.size() > 1) {
             throw new RuntimeException(
                     "found more than one (" + matchingConstructors.size() + ") matching constructor for " +

virtdata-api/src/test/java/io/nosqlbench/virtdata/core/bindings/ArgsComparatorTest.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.core.bindings;
+
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Constructor;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ArgsComparatorTest {
+
+
+    static Constructor<?> ctor_longBoxed = LongBoxed.class.getConstructors()[0];
+    static Constructor<?> ctor_longs = Longs.class.getConstructors()[0];
+    static Constructor<?> ctor_mixed = Mixed.class.getConstructors()[0];
+    static Constructor<?> ctor_boxed = Boxed.class.getConstructors()[0];
+    static Constructor<?> ctor_primitives = Primitives.class.getConstructors()[0];
+
+
+    @Test
+    public void verifyRanks() {
+        Object[] args = {1, 2};
+        ArgsComparator comparator = new ArgsComparator(args);
+        assertThat(comparator.matchRank(ctor_longBoxed,args)).isEqualTo(ArgsComparator.MATCHRANK.INCOMPATIBLE);
+        assertThat(comparator.matchRank(ctor_longs,args)).isEqualTo(ArgsComparator.MATCHRANK.BOXED);
+        assertThat(comparator.matchRank(ctor_mixed,args)).isEqualTo(ArgsComparator.MATCHRANK.DIRECT);
+        assertThat(comparator.matchRank(ctor_boxed,args)).isEqualTo(ArgsComparator.MATCHRANK.DIRECT);
+        assertThat(comparator.matchRank(ctor_primitives,args)).isEqualTo(ArgsComparator.MATCHRANK.DIRECT);
+    }
+
+    @Test
+    public void testCtorSanity() {
+        Object[] args = {1, 2};
+        try {
+            ctor_boxed.newInstance(args);
+            ctor_longs.newInstance(args);
+            ctor_mixed.newInstance(args);
+            ctor_primitives.newInstance(args);
+//            ctor_longBoxed.newInstance(args);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private static class Primitives {
+        public Primitives(int a, int b) {
+        }
+    }
+
+    private static class Boxed {
+        public Boxed(Integer a, Integer b) {
+        }
+    }
+
+    private static class Mixed {
+        public Mixed(int a, Integer b) {
+        }
+    }
+
+    private static class Longs {
+        public Longs(long a, long b) {
+        }
+    }
+
+    private static class LongBoxed {
+        public LongBoxed(Long a, Long b) {}
+    }
+
+
+}

virtdata-api/src/test/java/io/nosqlbench/virtdata/core/bindings/VirtDataFunctionResolverTest.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.core.bindings;
+
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class VirtDataFunctionResolverTest {
+
+    @Test
+    public void testArgSorting() {
+        ArgsComparator comparator =
+            new ArgsComparator(new Object[]{1, 2});
+        ArrayList<Constructor<?>> ctors = new ArrayList<>();
+        Constructor<?> long_boxed = LongBoxed.class.getConstructors()[0];
+        Constructor<?> longs = Longs.class.getConstructors()[0];
+        Constructor<?> mixed = Mixed.class.getConstructors()[0];
+        Constructor<?> boxed = Boxed.class.getConstructors()[0];
+        Constructor<?> primitives = Primitives.class.getConstructors()[0];
+
+        ctors.add(long_boxed);
+        ctors.add(longs);
+        ctors.add(mixed);
+        ctors.add(boxed);
+        ctors.add(primitives);
+
+        Collections.sort(ctors,comparator);
+        assertThat(ctors).containsExactly(mixed, boxed, primitives,longs,long_boxed);
+    }
+
+    private static class Primitives {
+        public Primitives(int a, int b) {
+        }
+    }
+
+    private static class Boxed {
+        public Boxed(Integer a, Integer b) {
+        }
+    }
+
+    private static class Mixed {
+        public Mixed(int a, Integer b) {
+        }
+    }
+
+    private static class Longs {
+        public Longs(long a, long b) {
+        }
+    }
+
+    private static class LongBoxed {
+        public LongBoxed(Long a, Long b) {}
+    }
+}