Mwolters/metadata (#2166)

* add processing of data type metadata

* switching internal classes to nbvector lib

* cleanup and adding comments
This commit is contained in:
Mark Wolters authored and GitHub committed 2025-03-11 08:34:36 -05:00
1 parent 30aa883c8b
commit 55bec2d65f
14 files changed
+212 -430

No files matched your search

+6
View File
@@ -45,6 +45,12 @@
<version>0.9.0</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nbvectors</artifactId>
<version>0.1.1</version>
</dependency>
</dependencies>
<build>
@@ -1,23 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
import java.nio.ByteBuffer;
public interface BBWriter<T> {
ByteBuffer encode(ByteBuffer out);
}
@@ -1,71 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
import java.nio.ByteBuffer;
import java.util.Arrays;
public record ConjugateNode(ConjugateType type, PNode<?>... values)
implements BBWriter<ConjugateNode>, PNode<ConjugateNode>
{
public ConjugateNode(ByteBuffer b) {
this(ConjugateType.values()[b.get()], readValues(b));
}
private static PNode<?>[] readValues(ByteBuffer b) {
byte count = b.get();
PNode<?>[] elements = new PNode[count];
for (int i = 0; i < elements.length; i++) {
elements[i] = PNode.fromBuffer(b);
}
return elements;
}
@Override
public ByteBuffer encode(ByteBuffer out) {
out.put((byte) this.type.ordinal()).put((byte) this.values.length);
for (PNode<?> element : values) {
element.encode(out);
}
return out;
}
@Override
public String toString() {
final StringBuffer sb = new StringBuffer("ConjugateNode{");
sb.append("type=").append(type);
sb.append(", v=").append(values == null ? "null" : Arrays.asList(values).toString());
sb.append('}');
return sb.toString();
}
@Override
public boolean equals(Object o) {
if (!(o instanceof ConjugateNode(ConjugateType type1, PNode<?>[] values1)))
return false;
return Arrays.equals(values, values1) && type == type1;
}
@Override
public int hashCode() {
int result = type.hashCode();
result = 31 * result + Arrays.hashCode(values);
return result;
}
}
@@ -1,23 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
public enum ConjugateType {
PRED,
AND,
OR
}
@@ -1,22 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
import java.util.function.Function;
public interface NodeRepresenter extends Function<PNode<?>, String> {
}
@@ -1,37 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
public enum OpType {
GT(">"),
LT("<"),
EQ("="),
NE("!="),
GE(">="),
LE("<="),
IN("IN");
private final String symbol;
OpType(String symbol) {
this.symbol=symbol;
}
public String symbol() {
return this.symbol;
}
}
@@ -1,30 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
import java.nio.ByteBuffer;
public sealed interface PNode<T> extends BBWriter<T> permits ConjugateNode, PredicateNode {
public static PNode<?> fromBuffer(ByteBuffer b) {
byte typeOrdinal = b.get(b.position());
return switch(ConjugateType.values()[typeOrdinal]) {
case AND, OR -> new ConjugateNode(b);
case PRED -> new PredicateNode(b);
};
}
}
@@ -1,96 +0,0 @@
/*
* Copyright (c) 2025 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nbvectors.buildhdf5.predicates.types;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Objects;
public record PredicateNode(
/* encoded as byte */
int field,
/* encoded as byte */
OpType op,
/* encoded as short len, and then longs */
long... v
) implements BBWriter<PredicateNode>, PNode<PredicateNode>
{
public PredicateNode(byte type, int field, OpType op, long... v) {
this(field, op, v);
}
public PredicateNode(ByteBuffer b) {
this(
b.get(),
b.get(),
OpType.values()[b.get()],
readValues(b)
);
}
private static long[] readValues(ByteBuffer b) {
int len = b.getShort();
long[] values = new long[len];
for (int i = 0; i < values.length; i++) {
values[i] = b.getLong();
}
return values;
}
@Override
public ByteBuffer encode(ByteBuffer out) {
out.put((byte) ConjugateType.PRED.ordinal());
out.put((byte) field).put((byte) op.ordinal());
out.putShort((short) v.length);
for (long l : v) {
out.putLong(l);
}
return out;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof PredicateNode(int fieldOffset, OpType op1, long[] v1)))
return false;
return field == fieldOffset && Arrays.equals(v, v1) && op == op1;
}
@Override
public int hashCode() {
int result = field;
result = 31 * result + Objects.hashCode(op);
result = 31 * result + Arrays.hashCode(v);
return result;
}
@Override
public String toString() {
final StringBuffer sb = new StringBuffer("PredicateNode{");
sb.append("field=").append(field);
sb.append(", op=").append(op);
sb.append(", v=");
if (v == null)
sb.append("null");
else {
sb.append('[');
for (int i = 0; i < v.length; ++i)
sb.append(i == 0 ? "" : ", ").append(v[i]);
sb.append(']');
}
sb.append('}');
return sb.toString();
}
}
@@ -28,19 +28,40 @@ import org.apache.logging.log4j.Logger;
import java.util.Map;
/**
* Abstract base class for reading data from HDF5 files.
* Provides common functionality for accessing and processing HDF5 datasets and their attributes.
*/
public abstract class AbstractHdfFileToVectorType implements HdfAttributesProcessor {
protected final HdfFile hdfFile;
protected final Dataset dataset;
protected final int[] dims;
private final static Logger logger = LogManager.getLogger(AbstractHdfFileToVectorType.class);
protected final HdfFile hdfFile; // The HDF5 file handle
protected final Dataset dataset; // The specific dataset within the HDF5 file
protected final String datasetName; // Name/path of the dataset
protected final int[] dims; // Dimensions of the dataset
protected String dataType; // Data type of the dataset
protected final static Logger logger = LogManager.getLogger(AbstractHdfFileToVectorType.class);
/**
* Constructs a new instance for reading data from an HDF5 file.
*
* @param filename The name of the HDF5 file to read
* @param datasetName The path to the dataset within the HDF5 file
*/
public AbstractHdfFileToVectorType(String filename, String datasetName) {
hdfFile = new HdfFile(NBIO.all().search(filename).one().asPath());
processAttributes(hdfFile);
this.datasetName = datasetName;
dataset = hdfFile.getDatasetByPath(datasetName);
dims = dataset.getDimensions();
processAttributes(hdfFile);
this.dataType = (dataType == null) ? dataset.getJavaType().getSimpleName().toLowerCase() : dataType;
}
/**
* Retrieves data from the dataset at a specific position.
* For multi-dimensional datasets, returns a slice of data.
*
* @param l The position to read from
* @return The data at the specified position
*/
protected Object getDataFrom(long l) {
long[] sliceOffset = new long[dims.length];
sliceOffset[0] = (l % dims[0]);
@@ -55,14 +76,24 @@ public abstract class AbstractHdfFileToVectorType implements HdfAttributesProces
}
}
/**
* Processes attributes of the HDF5 file by traversing its node hierarchy.
*
* @param hdfFile The HDF5 file to process
*/
@Override
public void processAttributes(HdfFile hdfFile) {
processNode(hdfFile);
}
private static void processNode(Node node) {
/**
* Recursively processes a node and its children in the HDF5 file hierarchy.
*
* @param node The node to process
*/
private void processNode(Node node) {
logger.info("Node: {}", node.getPath());
printAttributes(node);
readAttributes(node);
// If the node is a group, iterate through its children
if (node instanceof Group group) {
@@ -72,7 +103,13 @@ public abstract class AbstractHdfFileToVectorType implements HdfAttributesProces
}
}
private static void printAttributes(Node node) {
/**
* Reads and processes attributes of a node, updating the dataType if a
* COMPONENT_ENCODING attribute is found for the target dataset.
*
* @param node The node whose attributes should be read
*/
private void readAttributes(Node node) {
Map<String, Attribute> attributes = node.getAttributes();
if (attributes.isEmpty()) {
logger.info(() -> "No attributes");
@@ -80,6 +117,9 @@ public abstract class AbstractHdfFileToVectorType implements HdfAttributesProces
logger.info(() -> "Attributes:");
for (Map.Entry<String, Attribute> entry : attributes.entrySet()) {
logger.info("{} = {}", entry.getKey(), entry.getValue().getData());
if (entry.getKey().equals(COMPONENT_ENCODING) && (node.getPath().equalsIgnoreCase(datasetName))) {
dataType = entry.getValue().getData().toString();
}
}
}
}
@@ -19,8 +19,29 @@ package io.nosqlbench.virtdata.library.hdf5.helpers;
import io.jhdf.HdfFile;
/**
* Interface for processing attributes from HDF5 files.
* Defines constants for common HDF5 dataset and attribute names used in nosqlbench contexts.
*/
public interface HdfAttributesProcessor {
/** Number of nearest neighbors */
public static final String NEIGHBORS = "neighbors";
/** Dimensionality of the vector space */
public static final String DIMENSIONS = "dimensions";
/** Training dataset vectors */
public static final String TRAIN_VECTORS = "train_vectors";
/** Test dataset vectors */
public static final String TEST_VECTORS = "test_vectors";
/** Machine learning model information */
public static final String MODEL = "model";
/** Distance metric used for similarity calculations */
public static final String DISTANCE_FUNCTION = "distance_function";
/** Encoding format for vector components */
public static final String COMPONENT_ENCODING = "component_encoding";
/**
* Process attributes from an HDF5 file.
* @param hdfFile The HDF5 file to process attributes from
*/
public void processAttributes(HdfFile hdfFile);
}
@@ -21,66 +21,104 @@ import io.nosqlbench.nbvectors.buildhdf5.predicates.types.NodeRepresenter;
import io.nosqlbench.nbvectors.buildhdf5.predicates.types.PNode;
import io.nosqlbench.nbvectors.buildhdf5.predicates.types.PredicateNode;
/**
* Renders predicate nodes into CQL-compatible string representations.
* This class implements NodeRepresenter to convert different types of nodes
* into their corresponding CQL syntax.
*/
public class CqlNodeRenderer implements NodeRepresenter {
/** Schema field names used for rendering */
private final String[] schema;
/** Constant for space character used in string building */
public static final String SPACE = " ";
/**
* Creates a new CQL node renderer with the specified schema.
* @param schema Array of field names used for rendering predicates
*/
public CqlNodeRenderer(String[] schema) {
this.schema = schema;
}
/**
* Converts a predicate node into its CQL string representation.
* @param node The node to convert
* @return CQL string representation of the node
*/
@Override
public String apply(PNode<?> node) {
return switch (node) {
case ConjugateNode n -> reprConjugate(n);
case PredicateNode p -> reprPredicate(p);
};
}
private String reprPredicate(PredicateNode p) {
StringBuilder sb = new StringBuilder();
sb.append(schema[p.field()]);
if (isChar(p.op().symbol())) {
sb.append(" ");
public String apply(PNode<?> node) {
return switch (node) {
case ConjugateNode n -> reprConjugate(n);
case PredicateNode p -> reprPredicate(p);
};
}
sb.append(p.op().symbol());
if (p.v().length>1) {
sb.append("(");
String delim="";
for (long v : p.v()) {
sb.append(delim);
sb.append(v);
delim=",";
}
sb.append(")");
} else {
sb.append(p.v()[0]);
/**
* Renders a predicate node into CQL syntax.
* @param p The predicate node to render
* @return CQL representation like "field_name operator value"
*/
private String reprPredicate(PredicateNode p) {
StringBuilder sb = new StringBuilder();
sb.append(schema[p.field()]);
if (isChar(p.op().symbol())) {
sb.append(SPACE);
}
sb.append(p.op().symbol());
if (p.v().length>1) {
sb.append("(");
String delim="";
for (long v : p.v()) {
sb.append(delim);
sb.append(v);
delim=",";
}
sb.append(")");
} else {
sb.append(p.v()[0]);
}
return sb.toString();
}
return sb.toString();
}
private boolean isChar(String symbol) {
char c = symbol.charAt(0);
return (c >= 'A' && c <= 'Z') || (c>='a' && c<='z');
}
/**
* Checks if the first character of a symbol is alphabetic.
* @param symbol The symbol to check
* @return true if first character is a letter, false otherwise
*/
private boolean isChar(String symbol) {
char c = symbol.charAt(0);
return (c >= 'A' && c <= 'Z') || (c>='a' && c<='z');
}
private String reprConjugate(ConjugateNode n) {
return switch (n.type()) {
case AND,OR -> concatenate(n.type().name(),n.values());
case PRED -> throw new RuntimeException("impossible unless broken code");
};
}
/**
* Renders a conjugate node (AND/OR combinations) into CQL syntax.
* @param n The conjugate node to render
* @return CQL representation of the conjugated predicates
*/
private String reprConjugate(ConjugateNode n) {
return switch (n.type()) {
case AND,OR -> concatenate(n.type().name(),n.values());
case PRED -> throw new RuntimeException("impossible unless broken code");
};
}
private String concatenate(String name, PNode<?>[] values) {
StringBuilder sb = new StringBuilder();
/**
* Concatenates multiple nodes with the specified operator.
* @param name The operator name (AND/OR)
* @param values The nodes to concatenate
* @return Space-separated string of nodes joined by the operator
*/
private String concatenate(String name, PNode<?>[] values) {
StringBuilder sb = new StringBuilder();
for (PNode<?> value : values) {
String nodeRep = apply(value);
if (!sb.isEmpty()) {
sb.append(" ").append(name).append(" ");
}
sb.append(nodeRep);
}
return sb.toString();
}
for (PNode<?> value : values) {
String nodeRep = apply(value);
if (!sb.isEmpty()) {
sb.append(SPACE).append(name).append(SPACE);
}
sb.append(nodeRep);
}
return sb.toString();
}
}
@@ -23,46 +23,57 @@ import io.nosqlbench.nbvectors.buildhdf5.predicates.types.NodeRepresenter;
import io.nosqlbench.nbvectors.buildhdf5.predicates.types.PNode;
import io.nosqlbench.nbvectors.buildhdf5.predicates.types.PredicateNode;
/**
* Renders HDF5 predicate nodes as JSON strings.
* This class implements NodeRepresenter to provide JSON string representations
* of predicate nodes used in HDF5 operations.
*/
public class H5JsonNodeRenderer implements NodeRepresenter {
/** Schema information used for rendering */
private final String[] schema;
/** Gson instance configured for pretty printing */
private final static Gson gson = new GsonBuilder().setPrettyPrinting().create();
/**
* Creates a new renderer with the specified schema.
*
* @param schema Array of strings representing the schema structure
*/
public H5JsonNodeRenderer(String[] schema) {
this.schema = schema;
}
/**
* Converts a predicate node to its JSON string representation.
*
* @param node The predicate node to render
* @return JSON string representation of the node
*/
@Override
public String apply(PNode<?> node) {
return switch (node) {
case ConjugateNode n -> renderConjugate(n);
case PredicateNode p -> renderPredicate(p);
};
}
public String apply(PNode<?> node) {
return switch (node) {
case ConjugateNode n -> renderConjugate(n);
case PredicateNode p -> renderPredicate(p);
};
}
private String renderPredicate(PredicateNode p) {
return gson.toJson(p);
}
private boolean isChar(String symbol) {
char c = symbol.charAt(0);
return (c >= 'A' && c <= 'Z') || (c>='a' && c<='z');
}
private String renderConjugate(ConjugateNode n) {
return gson.toJson(n);
}
private String concatenate(String name, PNode<?>[] values) {
StringBuilder sb = new StringBuilder();
for (PNode<?> value : values) {
String nodeRep = apply(value);
if (!sb.isEmpty()) {
sb.append(" ").append(name).append(" ");
}
sb.append(nodeRep);
}
return sb.toString();
}
/**
* Renders a predicate node as JSON.
*
* @param p The predicate node to render
* @return JSON string representation of the predicate
*/
private String renderPredicate(PredicateNode p) {
return gson.toJson(p);
}
/**
* Renders a conjugate node as JSON.
*
* @param n The conjugate node to render
* @return JSON string representation of the conjugate
*/
private String renderConjugate(ConjugateNode n) {
return gson.toJson(n);
}
}
@@ -36,15 +36,15 @@ public class PredicateTest {
System.out.println(p1);
}
// @Test
// public void testNesting() {
// Node tn = getTestNode1();
// ByteBuffer b = ByteBuffer.allocate(100);
// tn.encode(b);
// b.flip();
// Node<?> node = new ConjugateNode(b);
// System.out.println(node);
// }
@Test
public void testNesting() {
PNode tn = getTestNode1();
ByteBuffer b = ByteBuffer.allocate(100);
tn.encode(b);
b.flip();
PNode<?> node = new ConjugateNode(b);
System.out.println(node);
}
private PNode<?> getTestNode1() {
PredicateNode inp = new PredicateNode(1, OpType.IN, new long[]{3L, 4L});
@@ -109,36 +109,4 @@ public class H5JsonNodeRendererTest {
]
}""";
public static final String testExample = """
{
"conjunction": "none",
"terms": [
{
"field": {"name": "firstname"},
"operator": "in",
"comparator": {"value":
["Mark","Mark's friend Joe"]
}
}
]
}
""";
public static final String test3 = """
{
"conjunction": "or",
"terms": [
{
"field": {"name": "highprice"},
"operator": "gt",
"comparator": {"value": 1000}
},
{
"field": {"name": "lowprice"},
"operator": "lt",
"comparator": {"value": 1}
}
]
}
""";
}