Mwolters/pred enhancements (#2159)

* adding new binding for single array int types

* new binding for var length int arrays

* new binding for var length int arrays cleanup

* add embedding to convert int array to string

* add script for hdf5 generation

* list version of var length int embedding

* updating target byte code for groovy scripts

* adding pytorch version

* removing datagen files to put them in tools repo

* make IntArrayToString threadsafe

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/AbstractHdfFileToVectorType.java

@@ -36,10 +36,13 @@ public abstract class AbstractHdfFileToVectorType {
         long[] sliceOffset = new long[dims.length];
         sliceOffset[0] = (l % dims[0]);
         int[] sliceDimensions = new int[dims.length];
-        // We always want to read a single vector
+        // We always want to read a single value
         sliceDimensions[0] = 1;
-        // Number of elements in the vector
+        if (dims.length > 1) {
-        sliceDimensions[1] = dims[1];
+            sliceDimensions[1] = dims[1];
-        return dataset.getData(sliceOffset, sliceDimensions);
+            return dataset.getData(sliceOffset, sliceDimensions);
+        } else {
+            return dataset.getData();
+        }
     }
 }

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_array/HdfFileToVarLengthIntArray.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_array;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.hdf5.from_long.AbstractHdfFileToVectorType;
+
+import java.util.function.LongFunction;
+
+/**
+ * This function reads a vector dataset from an HDF5 file. The dataset itself is not
+ * read into memory, only the metadata (the "dataset" Java Object). The lambda function
+ * reads a single vector from the dataset, based on the long input value. As currently
+ * written this class will only work for datasets with 2 dimensions where the 1st dimension
+ * specifies the number of vectors and the 2nd dimension specifies the number of elements in
+ * each vector. Only datatypes short, int, and float are supported at this time.
+ * <p>
+ * This implementation is specific to returning an array of ints from a dataset that contains
+ * variable length arrays of ints.
+ */
+@ThreadSafeMapper
+@Categories(Category.experimental)
+public class HdfFileToVarLengthIntArray extends AbstractHdfFileToVectorType implements LongFunction<int[]> {
+
+    public HdfFileToVarLengthIntArray(String filename, String datasetName) {
+        super(filename, datasetName);
+    }
+    @Override
+    public int[] apply(long l) {
+        Object data = getDataFrom(l);
+        return extractIds(data, l);
+    }
+
+    /**
+     * @param o The dataset from hdf5 file. In this case because the number of ints in each array is variable length it
+     *          is returned by hdf5 as a one dimensional array of Objects
+     * @param l The current cycle
+     * @return The int array at the offset of l
+     */
+    private int[] extractIds(Object o, long l) {
+        Object[] objects = (Object[]) o;
+        Object o1 = objects[(int) l % objects.length];
+        return (int[]) o1;
+    }
+
+}

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_int/HdfFileToInt.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_int;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.hdf5.from_long.AbstractHdfFileToVectorType;
+import io.nosqlbench.virtdata.library.hdf5.helpers.EmbeddingGenerator;
+import io.nosqlbench.virtdata.library.hdf5.helpers.EmbeddingGeneratorFactory;
+
+import java.util.function.LongFunction;
+
+/**
+ * This function reads a vector dataset from an HDF5 file. The dataset itself is not
+ * read into memory, only the metadata (the "dataset" Java Object). The lambda function
+ * reads a single vector from the dataset, based on the long input value. As currently
+ * written this class will only work for datasets with 2 dimensions where the 1st dimension
+ * specifies the number of vectors and the 2nd dimension specifies the number of elements in
+ * each vector. Only datatypes short, int, and float are supported at this time.
+ * <p>
+ * This implementation is specific to returning a single int
+ */
+@ThreadSafeMapper
+@Categories(Category.experimental)
+public class HdfFileToInt extends AbstractHdfFileToVectorType implements LongFunction<Integer> {
+    private final EmbeddingGenerator embeddingGenerator;
+
+    public HdfFileToInt(String filename, String datasetName) {
+        super(filename, datasetName);
+        embeddingGenerator = EmbeddingGeneratorFactory.getGenerator(dataset.getJavaType().getSimpleName().toLowerCase());
+    }
+    @Override
+    public Integer apply(long l) {
+        Object data = getDataFrom(l);
+        return embeddingGenerator.generateIntFrom(data, l);
+    }
+
+}

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_list/HdfFileToVarLengthIntList.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_list;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.hdf5.from_long.AbstractHdfFileToVectorType;
+
+import java.util.List;
+import java.util.function.LongFunction;
+
+/**
+ * This function reads a vector dataset from an HDF5 file. The dataset itself is not
+ * read into memory, only the metadata (the "dataset" Java Object). The lambda function
+ * reads a single vector from the dataset, based on the long input value. As currently
+ * written this class will only work for datasets with 2 dimensions where the 1st dimension
+ * specifies the number of vectors and the 2nd dimension specifies the number of elements in
+ * each vector. Only datatypes short, int, and float are supported at this time.
+ * <p>
+ * This implementation is specific to returning a List of type Integer from a dataset that contains
+ * variable length arrays of ints.
+ */
+@ThreadSafeMapper
+@Categories(Category.experimental)
+public class HdfFileToVarLengthIntList extends AbstractHdfFileToVectorType implements LongFunction<List<Integer>> {
+
+    public HdfFileToVarLengthIntList(String filename, String datasetName) {
+        super(filename, datasetName);
+    }
+    @Override
+    public List<Integer> apply(long l) {
+        Object data = getDataFrom(l);
+        return extractIds(data, l);
+    }
+
+    /**
+     * @param o The dataset from hdf5 file. In this case because the number of ints in each array is variable length it
+     *          is returned by hdf5 as a one dimensional array of Objects
+     * @param l The current cycle
+     * @return A List of the integers at the offset of l
+     */
+    private List<Integer> extractIds(Object o, long l) {
+        Object[] objects = (Object[]) o;
+        int[] ints = (int[]) objects[(int) l % objects.length];
+        Integer[] integers = new Integer[ints.length];
+        for (int i = 0; i < ints.length; i++) {
+            integers[i] = ints[i];
+        }
+        return List.of(integers);
+    }
+
+}

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_string/IntArrayToString.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_string;
+
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+
+import java.util.function.Function;
+
+@ThreadSafeMapper
+@Categories(Category.experimental)
+public class IntArrayToString implements Function<int[],String> {
+    private final ThreadLocal<StringBuilder> threadSb = ThreadLocal.withInitial(StringBuilder::new);
+
+    @Override
+    public String apply(int[] ints) {
+        StringBuilder sb = threadSb.get();
+        sb.setLength(0);
+        for (int i : ints) {
+            sb.append(i);
+            if (i != ints[ints.length - 1]) {
+                sb.append(",");
+            }
+        }
+        return sb.toString();
+    }
+}

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/helpers/DoubleEmbeddingGenerator.java

@@ -81,4 +81,10 @@ public class DoubleEmbeddingGenerator implements EmbeddingGenerator {
         return vector2;
     }
 
+    @Override
+    public int generateIntFrom(Object o, long l) {
+        double[] source = (double[]) o;
+        return (int) source[(int) (l % source.length)];
+    }
+
 }

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/helpers/EmbeddingGenerator.java

@@ -30,4 +30,6 @@ public interface EmbeddingGenerator {
     List<Integer> generateIntListEmbeddingFrom(Object data, int[] dims);
 
     int[] generateIntArrayEmbeddingFrom(Object data, int[] dims);
+
+    int generateIntFrom(Object data, long l);
 }

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/helpers/FloatEmbeddingGenerator.java

@@ -76,4 +76,10 @@ public class FloatEmbeddingGenerator implements EmbeddingGenerator {
         return vector2;
     }
 
+    @Override
+    public int generateIntFrom(Object o, long l) {
+        float[] source = (float[]) o;
+        return (int) source[(int) (l % source.length)];
+    }
+
 }

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/helpers/IntEmbeddingGenerator.java

@@ -74,4 +74,10 @@ public class IntEmbeddingGenerator implements EmbeddingGenerator {
     public int[] generateIntArrayEmbeddingFrom(Object o, int[] dims) {
         return ((int[][]) o)[0];
     }
+
+    @Override
+    public int generateIntFrom(Object o, long l) {
+        int[] source = (int[]) o;
+        return source[(int) (l % source.length)];
+    }
 }

nb-virtdata/virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/helpers/LongEmbeddingGenerator.java

@@ -75,4 +75,10 @@ public class LongEmbeddingGenerator implements EmbeddingGenerator {
         }
         return vector2;
     }
+
+    @Override
+    public int generateIntFrom(Object o, long l) {
+        long[] source = (long[]) o;
+        return Math.toIntExact(source[(int) (l % source.length)]);
+    }
 }