mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2026-10-12 09:24:52 -05:00
feat(vectordata): read format version 3 and hold an absent version to 1
Version 3 is stated parents and a profile tag schema: every profile but default names a real parent or the load is refused naming the profiles involved, and the schema loads in naming order. An unversioned manifest is read as the least it could be and refused, naming the version to declare, when its content needs more. The axis of an inheritance step is derived from base_count rather than the parent's name: base facets are re-cut to the child's count, the per-size outputs, metadata_results now among them, do not cross a size step, and base_count and maxk inherit as the reference's do. A parent YAML reads as a number is the name it spells. Test fixtures that declare a series now state version 2.
This commit is contained in:
Jonathan Shook committed
1 parent
4a1f6a4d06
commit
9f3efebe85
11 files changed
+590
-79
No files matched your search
@@ -16,10 +16,14 @@
|
||||
package io.nosqlbench.vectordata;
|
||||
|
||||
/// The `dataset.yaml` format version: a **minimum reader requirement**,
|
||||
/// not a timestamp. Absent means [#BASE], which is every dataset in
|
||||
/// circulation and not a distinct unversioned state. A writer emits the
|
||||
/// lowest version describing what it wrote, so only a change that older
|
||||
/// readers would misread — multi-file facets were the first — bumps it.
|
||||
/// not a timestamp. Absent means [#BASE], and the dataset is **held to
|
||||
/// it**: an unversioned manifest is read as what every dataset was
|
||||
/// before the field existed — single-file facets, one implicit parent,
|
||||
/// no tag schema — and content that needs more is refused naming the
|
||||
/// version to declare. A writer emits the lowest version describing
|
||||
/// what it wrote, so only a change that older readers would misread
|
||||
/// bumps it: multi-file facets were the first, stated parents and
|
||||
/// profile tag schemas the second.
|
||||
public final class FormatVersion {
|
||||
private FormatVersion() { }
|
||||
|
||||
@@ -27,8 +31,11 @@ public final class FormatVersion {
|
||||
public static final int BASE = 1;
|
||||
/// Multi-file facet series.
|
||||
public static final int SHARDED = 2;
|
||||
/// Profiles that state their parents, a parent other than `default`
|
||||
/// among them, and carry a `profile_tags` schema.
|
||||
public static final int TAGGED = 3;
|
||||
/// The highest version this implementation reads.
|
||||
public static final int SUPPORTED = SHARDED;
|
||||
public static final int SUPPORTED = TAGGED;
|
||||
|
||||
/// Refuses a dataset this implementation cannot read, naming both
|
||||
/// numbers, and returns the effective version otherwise. Shared by
|
||||
@@ -42,13 +49,23 @@ public final class FormatVersion {
|
||||
return version;
|
||||
}
|
||||
|
||||
/// Refuses a declaration that understates what it holds: a stated
|
||||
/// Refuses a declaration that understates what it holds. A stated
|
||||
/// version lower than the content requires is a declaration
|
||||
/// contradicting itself, the same class of fault as a record count
|
||||
/// that disagrees with its shards. An absent field is not a claim
|
||||
/// and passes; a version higher than needed is merely generous.
|
||||
/// that disagrees with its shards. An **absent** field means
|
||||
/// [#BASE] and the dataset is held to it: content that needs more
|
||||
/// is refused naming the version to declare, so a file that never
|
||||
/// said what it is is read as the least it could be rather than the
|
||||
/// most a new reader can make of it. A version higher than needed
|
||||
/// is merely generous.
|
||||
public static void checkStatedAgainstContent(Integer stated, int required) {
|
||||
if (stated != null && required > stated)
|
||||
if (stated == null) {
|
||||
if (required > BASE)
|
||||
throw new VectorDataException("dataset declares no format_version and carries content that requires "
|
||||
+ required + "; declare format_version: " + required);
|
||||
return;
|
||||
}
|
||||
if (required > stated)
|
||||
throw new VectorDataException("dataset declares format_version " + stated + " but its content requires "
|
||||
+ required + " — a declaration cannot understate what it holds");
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ package io.nosqlbench.vectordata;
|
||||
|
||||
import io.nosqlbench.vectordata.internal.HttpTransport;
|
||||
import io.nosqlbench.vectordata.internal.ManifestView;
|
||||
import io.nosqlbench.vectordata.internal.ProfileParents;
|
||||
import io.nosqlbench.vectordata.internal.Shards;
|
||||
import io.nosqlbench.vectordata.internal.SourceSpec;
|
||||
import io.nosqlbench.vectordata.internal.YamlData;
|
||||
@@ -26,6 +27,7 @@ import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
@@ -37,27 +39,44 @@ import java.util.Set;
|
||||
/// Profiles name only what differs and inherit the rest. A non-default
|
||||
/// profile inherits unstated facets from the profile it names with
|
||||
/// `inherits:` (or the older `extends:`), else from `default` — and
|
||||
/// **what inherits depends on the axis**. Across the size axis (parent
|
||||
/// `default`), `base_vectors` and `metadata_content` inherit under the
|
||||
/// child's `base_count` window, while the neighbor facets do not: ground
|
||||
/// truth is derived from `base_count`, so a sized profile that omits its
|
||||
/// own fails loudly rather than reading the full base's. Across any
|
||||
/// other axis every facet is invariant and inherits as is. A
|
||||
/// **what inherits depends on the axis**, which is derived from
|
||||
/// `base_count`: a step is on the size axis when the child's count
|
||||
/// differs from its parent's effective one, whatever the parent is
|
||||
/// called. Across the size axis `base_vectors` and `metadata_content`
|
||||
/// inherit **re-cut** to the child's `base_count` window, while the
|
||||
/// per-size outputs — the neighbor facets and `metadata_results` — do
|
||||
/// not: they are derived from `base_count`, so a sized profile that
|
||||
/// omits its own fails loudly rather than reading the full base's.
|
||||
/// Across a step at one size every facet is invariant and inherits as
|
||||
/// is, and so does `base_count`; `maxk` crosses every step. A
|
||||
/// `partition: true` profile is an oracle partition with independent
|
||||
/// base vectors, not a windowed subset, and inherits nothing.
|
||||
///
|
||||
/// From `format_version` 3 every profile other than `default` states
|
||||
/// its parent, and an absent, unknown, self, or cyclic parent — or a
|
||||
/// partition that names one — is refused at load naming the profiles
|
||||
/// involved. Below 3 an unknown or self parent falls back to `default`
|
||||
/// and a cycle leaves its members with what they declare.
|
||||
public final class TestDataGroup {
|
||||
private static final Set<String> NON_FACETS = Set.of("extends", "inherits", "base_count", "query_count", "maxk", "partition",
|
||||
"attributes", "name", "description", "tags");
|
||||
/// Per-profile outputs that do not cross the size axis.
|
||||
private static final Set<String> SIZE_AXIS_OUTPUTS = Set.of("neighbor_indices", "neighbor_distances",
|
||||
/// Per-profile outputs derived from `base_count`, which do not
|
||||
/// cross the size axis: a results index maps each predicate to base
|
||||
/// ordinals, so a parent's at another size is wrong for this
|
||||
/// profile in the same way its ground truth is.
|
||||
private static final Set<String> SIZE_AXIS_OUTPUTS = Set.of("neighbor_indices", "neighbor_distances", "metadata_results",
|
||||
"prefiltered_neighbor_indices", "prefiltered_neighbor_distances", "postfiltered_neighbor_indices", "postfiltered_neighbor_distances");
|
||||
/// Facets that inherit under the child's `base_count` window.
|
||||
private static final Set<String> WINDOWED_ON_INHERIT = Set.of("base_vectors", "metadata_content");
|
||||
|
||||
private final String name; private final URI manifest; private final Map<String, Map<String, Object>> profiles; private final VectorDataSettings settings;
|
||||
private final Map<String, Object> attributes;
|
||||
private final Map<String, Object> profileTags;
|
||||
private final int formatVersion;
|
||||
private TestDataGroup(String name, URI manifest, Map<String, Map<String, Object>> profiles, VectorDataSettings settings,
|
||||
Map<String, Object> attributes) {
|
||||
Map<String, Object> attributes, Map<String, Object> profileTags, int formatVersion) {
|
||||
this.name = name; this.manifest = manifest; this.profiles = profiles; this.settings = settings; this.attributes = attributes;
|
||||
this.profileTags = profileTags; this.formatVersion = formatVersion;
|
||||
}
|
||||
public static TestDataGroup load(String source) { return load(uri(source), VectorDataSettings.defaults()); }
|
||||
/** Builds a group from a legacy knn_entries layout whose sources are already resolved. */
|
||||
@@ -71,7 +90,8 @@ public final class TestDataGroup {
|
||||
profiles.put(entry.getKey(), profile);
|
||||
}
|
||||
if (profiles.isEmpty()) throw new VectorDataException("Legacy catalog has no profiles for " + name);
|
||||
return new TestDataGroup(name, origin, profiles, settings, Map.of());
|
||||
// Synthesized in memory: version 1 by construction.
|
||||
return new TestDataGroup(name, origin, profiles, settings, Map.of(), Map.of(), FormatVersion.BASE);
|
||||
}
|
||||
public static TestDataGroup load(URI source, VectorDataSettings settings) {
|
||||
if (!isYaml(source)) {
|
||||
@@ -95,7 +115,7 @@ public final class TestDataGroup {
|
||||
// anything else is read from it: the field exists to turn "no
|
||||
// such file" into a diagnosis.
|
||||
Integer stated = root.get("format_version") == null ? null : YamlData.integer(root.get("format_version"), "format_version");
|
||||
FormatVersion.checkSupported(stated);
|
||||
int version = FormatVersion.checkSupported(stated);
|
||||
String name = YamlData.optionalString(root.get("name")); if (name == null) name = basename(manifest);
|
||||
Object rawProfiles = root.get("profiles");
|
||||
Map<String, Map<String, Object>> profiles = new LinkedHashMap<>();
|
||||
@@ -107,7 +127,18 @@ public final class TestDataGroup {
|
||||
if (profiles.isEmpty()) throw new VectorDataException("Manifest contains no profiles: " + manifest);
|
||||
Map<String, Object> attributes = root.get("attributes") instanceof Map<?, ?> declared
|
||||
? Map.copyOf(YamlData.map(declared, "attributes")) : Map.of();
|
||||
TestDataGroup group = new TestDataGroup(name, manifest, profiles, settings, attributes);
|
||||
// The tag schema, in naming order: a dropped or reordered tag
|
||||
// would silently rename every generated profile.
|
||||
Map<String, Object> profileTags = root.get("profile_tags") instanceof Map<?, ?> declared
|
||||
? Collections.unmodifiableMap(YamlData.map(declared, "profile_tags")) : Map.of();
|
||||
TestDataGroup group = new TestDataGroup(name, manifest, profiles, settings, attributes, profileTags, version);
|
||||
// Stated parents: at version 3 every parent is real and stated,
|
||||
// or the load is refused naming the profiles; below 3 the
|
||||
// fallbacks stand.
|
||||
List<ProfileParents.Facts> parents = new ArrayList<>();
|
||||
for (Map.Entry<String, Map<String, Object>> profile : profiles.entrySet())
|
||||
parents.add(new ProfileParents.Facts(profile.getKey(), isPartition(profile.getValue()), statedParent(profile.getKey(), profile.getValue())));
|
||||
ProfileParents.check(version, parents);
|
||||
// The version the content requires is derived by folding every
|
||||
// declaration, never asserted — and every declaration is
|
||||
// checked for self-consistency here, before a facet opens.
|
||||
@@ -137,59 +168,101 @@ public final class TestDataGroup {
|
||||
}
|
||||
public String name() { return name; }
|
||||
public Map<String, Map<String, Object>> profiles() { return Map.copyOf(profiles); }
|
||||
/// The `format_version` this dataset states, or [FormatVersion#BASE]
|
||||
/// when it states none — exposed so a caller can decide before it
|
||||
/// fetches anything.
|
||||
public int formatVersion() { return formatVersion; }
|
||||
/// The profile tag schema, naming tags in order, each mapped to its
|
||||
/// default or to `null` for a naming tag. Empty when the dataset
|
||||
/// declares none.
|
||||
public Map<String, Object> profileTags() { return profileTags; }
|
||||
/// Opens a profile by its exact name. A `null` or blank name opens
|
||||
/// `default`, or the first profile when there is none.
|
||||
public TestDataView profile(String profile) {
|
||||
String selected = profile == null || profile.isBlank() ? (profiles.containsKey("default") ? "default" : profiles.keySet().iterator().next()) : profile;
|
||||
return new ManifestView(name, selected, facets(selected), settings, attributes);
|
||||
return new ManifestView(name, selected, resolve(selected).facets(), settings, attributes);
|
||||
}
|
||||
/// The format version this manifest's content requires: sharded if
|
||||
/// any profile declares a multi-file facet, base otherwise.
|
||||
/// The format version this manifest's content requires: tagged if
|
||||
/// it declares a tag schema or a profile names a parent other than
|
||||
/// `default`, sharded if any profile declares a multi-file facet,
|
||||
/// base otherwise.
|
||||
private int requiredVersion() {
|
||||
boolean sharded = false;
|
||||
for (Map.Entry<String, Map<String, Object>> profile : profiles.entrySet())
|
||||
for (FacetDescriptor facet : declared(profile.getKey(), profile.getValue()).values())
|
||||
if (facet.isSeries()) return FormatVersion.SHARDED;
|
||||
return FormatVersion.BASE;
|
||||
if (facet.isSeries()) sharded = true;
|
||||
boolean tagged = !profileTags.isEmpty();
|
||||
for (Map.Entry<String, Map<String, Object>> profile : profiles.entrySet()) {
|
||||
String parent = statedParent(profile.getKey(), profile.getValue());
|
||||
if (!"default".equals(profile.getKey()) && parent != null && !"default".equals(parent)) tagged = true;
|
||||
}
|
||||
return tagged ? FormatVersion.TAGGED : sharded ? FormatVersion.SHARDED : FormatVersion.BASE;
|
||||
}
|
||||
private Map<String, FacetDescriptor> facets(String profile) {
|
||||
/// A profile as loaded: its facets after inheritance, and the
|
||||
/// `base_count` and `maxk` it ends up with.
|
||||
private record Resolved(Map<String, FacetDescriptor> facets, Long baseCount, Integer maxk) { }
|
||||
private Resolved resolve(String profile) {
|
||||
Map<String, Object> definition = profiles.get(profile);
|
||||
if (definition == null) throw new VectorDataException("Dataset " + name + " has no profile " + profile);
|
||||
Map<String, FacetDescriptor> own = declared(profile, definition);
|
||||
Long ownCount = longOrNull(definition.get("base_count"), "base_count");
|
||||
Integer ownMaxk = definition.get("maxk") == null ? null : YamlData.integer(definition.get("maxk"), "maxk");
|
||||
// A cycle — `a` inherits `b`, `b` inherits `a` — never settles,
|
||||
// so its members, and anything inheriting from them, keep only
|
||||
// what they declare.
|
||||
Set<String> lineage = new HashSet<>();
|
||||
for (String at = profile; at != null; at = parentOf(at, profiles.get(at))) if (!lineage.add(at)) return own;
|
||||
for (String at = profile; at != null; at = parentOf(at, profiles.get(at))) if (!lineage.add(at)) return new Resolved(own, ownCount, ownMaxk);
|
||||
String parentName = parentOf(profile, definition);
|
||||
if (parentName == null) return own;
|
||||
Map<String, FacetDescriptor> parent = facets(parentName);
|
||||
boolean sizeAxis = "default".equals(parentName);
|
||||
Long baseCount = longOrNull(definition.get("base_count"), "base_count");
|
||||
if (parentName == null) return new Resolved(own, ownCount, ownMaxk);
|
||||
Resolved parent = resolve(parentName);
|
||||
// The axis of a step is derived from `base_count`: a size step
|
||||
// when the child's count differs from the parent's effective
|
||||
// one, whatever the parent is called; otherwise the step is at
|
||||
// one size and every facet crosses as it is.
|
||||
boolean sizeAxis = ownCount != null && !ownCount.equals(parent.baseCount());
|
||||
Map<String, FacetDescriptor> merged = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, FacetDescriptor> inherited : parent.entrySet()) {
|
||||
for (Map.Entry<String, FacetDescriptor> inherited : parent.facets().entrySet()) {
|
||||
String facetName = inherited.getKey();
|
||||
if (own.containsKey(facetName)) continue;
|
||||
if (sizeAxis && SIZE_AXIS_OUTPUTS.contains(facetName)) continue;
|
||||
merged.put(facetName, WINDOWED_ON_INHERIT.contains(facetName) ? inheritWithWindow(inherited.getValue(), baseCount) : inherited.getValue());
|
||||
merged.put(facetName, WINDOWED_ON_INHERIT.contains(facetName) ? inheritWithWindow(inherited.getValue(), ownCount) : inherited.getValue());
|
||||
}
|
||||
merged.putAll(own);
|
||||
return merged;
|
||||
return new Resolved(merged, ownCount != null ? ownCount : parent.baseCount(), ownMaxk != null ? ownMaxk : parent.maxk());
|
||||
}
|
||||
/// The profile a non-default profile inherits unstated facets from,
|
||||
/// or `null` when it inherits nothing: `default` itself, and a
|
||||
/// `partition: true` profile.
|
||||
/// `partition: true` profile. Below version 3 an unknown or self
|
||||
/// parent falls back to `default`; at 3 the loader has refused it.
|
||||
private String parentOf(String profile, Map<String, Object> definition) {
|
||||
if ("default".equals(profile)) return null;
|
||||
Object partition = definition.get("partition");
|
||||
if (Boolean.TRUE.equals(partition) || "true".equalsIgnoreCase(String.valueOf(partition))) return null;
|
||||
String named = YamlData.optionalString(definition.get("inherits"));
|
||||
if (named == null) named = YamlData.optionalString(definition.get("extends"));
|
||||
if (isPartition(definition)) return null;
|
||||
String named = statedParent(profile, definition);
|
||||
if (named != null && !named.equals(profile) && profiles.containsKey(named)) return named;
|
||||
return profiles.containsKey("default") ? "default" : null;
|
||||
}
|
||||
/// A base facet inherited across the size axis is windowed to the
|
||||
/// child's `base_count` unless it already carries a window of its
|
||||
/// own; without a `base_count` it is inherited as is.
|
||||
private static boolean isPartition(Map<String, Object> definition) {
|
||||
Object partition = definition.get("partition");
|
||||
return Boolean.TRUE.equals(partition) || "true".equalsIgnoreCase(String.valueOf(partition));
|
||||
}
|
||||
/// The parent a profile states under `inherits:` (or the older
|
||||
/// `extends:`), as written, or `null` for none. A name YAML read as
|
||||
/// a number — a rung such as `100` — comes back as the name it
|
||||
/// spells; anything else is refused.
|
||||
private static String statedParent(String profile, Map<String, Object> definition) {
|
||||
Object raw = definition.containsKey("inherits") ? definition.get("inherits") : definition.get("extends");
|
||||
if (raw == null) return null;
|
||||
if (raw instanceof String text) return text;
|
||||
if (raw instanceof Number number) return String.valueOf(number);
|
||||
throw new VectorDataException("profile " + profile + " inherits: expected a profile name, found " + raw);
|
||||
}
|
||||
/// A base facet inherited across the size axis is **re-cut** to the
|
||||
/// child's `base_count`: a window the parent carries is replaced,
|
||||
/// so a `20m` that builds on `10m` reads the first twenty million
|
||||
/// of the same file, not its parent's ten. Without a `base_count`
|
||||
/// it is inherited as is.
|
||||
private static FacetDescriptor inheritWithWindow(FacetDescriptor facet, Long baseCount) {
|
||||
if (baseCount == null || (facet.window() != null && !facet.window().isBlank())) return facet;
|
||||
if (baseCount == null) return facet;
|
||||
return new FacetDescriptor(facet.name(), facet.source(), "0.." + baseCount, facet.attributes(), facet.series(), facet.namespace());
|
||||
}
|
||||
/// The facets a profile declares itself, resolved against the
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Copyright (c) 2026 The NoSQLBench Authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.nosqlbench.vectordata.internal;
|
||||
|
||||
import io.nosqlbench.vectordata.FormatVersion;
|
||||
import io.nosqlbench.vectordata.VectorDataException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
/// Stated parents. From `format_version` 3 every profile other than
|
||||
/// `default` states `inherits:`, and a stated 3 is a claim that every
|
||||
/// parent is real and every profile has one meaning: an absent parent,
|
||||
/// an unknown or self parent, a cycle, and `partition: true` beside
|
||||
/// `inherits:` are load refusals naming the profiles involved. Versions
|
||||
/// 1 and 2 keep their fallbacks, and the same conditions are available
|
||||
/// on them as advisories phrased as what version 3 will refuse. One pure
|
||||
/// rule, so no two loaders can disagree.
|
||||
public final class ProfileParents {
|
||||
private ProfileParents() { }
|
||||
|
||||
/// What the rule reads of a profile: its name, whether it is a
|
||||
/// partition, and the parent it states, `null` for none.
|
||||
public record Facts(String name, boolean partition, String inherits) { }
|
||||
|
||||
/// Refuses a version-3 dataset whose parents are not all stated and
|
||||
/// real. Below 3 nothing is refused.
|
||||
public static void check(int version, List<Facts> profiles) {
|
||||
if (version < FormatVersion.TAGGED) return;
|
||||
List<String> faults = faults(profiles, true);
|
||||
if (!faults.isEmpty()) throw new VectorDataException(String.join("; ", faults));
|
||||
}
|
||||
|
||||
/// The same conditions on a dataset below version 3, phrased as
|
||||
/// what version 3 will refuse; empty at 3, where the loader refuses.
|
||||
public static List<String> advisories(int version, List<Facts> profiles) {
|
||||
if (version >= FormatVersion.TAGGED) return List.of();
|
||||
List<String> notes = new ArrayList<>();
|
||||
for (String fault : faults(profiles, false)) notes.add(fault + " (refused from format_version 3)");
|
||||
return notes;
|
||||
}
|
||||
|
||||
private static List<String> faults(List<Facts> profiles, boolean strict) {
|
||||
List<Facts> sorted = new ArrayList<>(profiles);
|
||||
sorted.sort(Comparator.comparing(Facts::name));
|
||||
Set<String> names = new HashSet<>();
|
||||
for (Facts profile : profiles) names.add(profile.name());
|
||||
List<String> faults = new ArrayList<>();
|
||||
for (Facts profile : sorted) {
|
||||
if ("default".equals(profile.name())) {
|
||||
if (profile.inherits() != null) faults.add("profile `default` is the base layer and names no parent");
|
||||
continue;
|
||||
}
|
||||
String parent = profile.inherits();
|
||||
if (profile.partition()) {
|
||||
if (parent != null) faults.add("profile `" + profile.name() + "` states both `partition: true` and `inherits:`; a partition builds on nothing");
|
||||
} else if (parent == null) {
|
||||
faults.add("profile `" + profile.name() + "` names no parent; state `inherits: default` or the layer it builds on");
|
||||
} else if (parent.equals(profile.name())) {
|
||||
faults.add("profile `" + profile.name() + "` names itself as its parent");
|
||||
} else if (!names.contains(parent)) {
|
||||
faults.add("profile `" + profile.name() + "` names an unknown parent `" + parent + "`");
|
||||
}
|
||||
}
|
||||
// A cycle is reported once, by its sorted members.
|
||||
Set<List<String>> cycles = new TreeSet<>(Comparator.comparing(List::toString));
|
||||
for (Facts profile : sorted) {
|
||||
if (profile.partition() || "default".equals(profile.name())) continue;
|
||||
List<String> seen = new ArrayList<>(List.of(profile.name()));
|
||||
String current = profile.name();
|
||||
while (true) {
|
||||
Facts at = find(profiles, current);
|
||||
if (at == null || at.partition() || "default".equals(at.name())) break;
|
||||
String parent = at.inherits();
|
||||
if (parent == null) { if (strict) break; parent = "default"; }
|
||||
if (parent.equals(profile.name())) {
|
||||
List<String> members = new ArrayList<>(seen);
|
||||
members.sort(Comparator.naturalOrder());
|
||||
cycles.add(members);
|
||||
break;
|
||||
}
|
||||
if (seen.contains(parent) || !names.contains(parent)) break;
|
||||
seen.add(parent);
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
for (List<String> members : cycles) {
|
||||
List<String> quoted = new ArrayList<>();
|
||||
for (String member : members) quoted.add("`" + member + "`");
|
||||
faults.add("profiles " + String.join(", ", quoted) + " close an inheritance cycle");
|
||||
}
|
||||
return faults;
|
||||
}
|
||||
|
||||
private static Facts find(List<Facts> profiles, String name) {
|
||||
for (Facts profile : profiles) if (profile.name().equals(name)) return profile;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -37,12 +37,17 @@ class CatalogManifestTest {
|
||||
FixtureSupport.fvec(dataset, "query.fvec", new float[][] {{3f, 4f}});
|
||||
FixtureSupport.ivec(dataset, "neighbors.ivecs", new int[][] {{9, 10}});
|
||||
FixtureSupport.ivec(dataset, "metadata.ivecs", new int[][] {{11, 12}});
|
||||
// A named parent other than `default` is what version 3 says,
|
||||
// and at 3 every profile states its parent.
|
||||
Files.writeString(dataset.resolve("dataset.yaml"), """
|
||||
format_version: 3
|
||||
name: sample
|
||||
profiles:
|
||||
parent:
|
||||
default:
|
||||
base: base.fvec
|
||||
custom_scores: {source: metadata.ivecs}
|
||||
parent:
|
||||
inherits: default
|
||||
demo:
|
||||
extends: parent
|
||||
query: {source: query.fvec, window: 0..1}
|
||||
|
||||
+2
@@ -343,6 +343,7 @@ class PrefetchRemoteIntegrationTest {
|
||||
|
||||
private static String uniformYaml(String base, int shards, int records, String extra) {
|
||||
return """
|
||||
format_version: 2
|
||||
name: remote-series
|
||||
profiles:
|
||||
default:
|
||||
@@ -366,6 +367,7 @@ class PrefetchRemoteIntegrationTest {
|
||||
@Test void anExplicitSeriesReadsOverHttp() throws Exception {
|
||||
String base = publishSeries(2, 25, false);
|
||||
VectorReader<float[]> reader = seriesView("explicit-http", """
|
||||
format_version: 2
|
||||
name: explicit
|
||||
profiles:
|
||||
default:
|
||||
|
||||
@@ -22,14 +22,20 @@ import org.junit.jupiter.api.io.TempDir;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/// What a profile inherits depends on the axis it varies along. Across
|
||||
/// the size axis, per-profile outputs cannot be shared — ground truth is
|
||||
/// derived from `base_count` — so a sized profile that omits its own
|
||||
/// neighbor facets fails loudly instead of reading the full base's.
|
||||
/// Across any other axis every facet is invariant and inherits as is.
|
||||
/// What a profile inherits depends on the axis it varies along, and the
|
||||
/// axis is derived from `base_count`: a step whose count differs from
|
||||
/// its parent's is a size step, whatever the parent is called. Across
|
||||
/// the size axis the base facets are re-cut to the child's count and
|
||||
/// the per-size outputs — ground truth and the results index — do not
|
||||
/// cross, so a sized profile that omits its own fails loudly instead of
|
||||
/// reading the full base's. Across a step at one size every facet is
|
||||
/// invariant and inherits as is. From `format_version` 3 every parent
|
||||
/// is stated and real, or the load is refused naming the profiles.
|
||||
@Tag("unit")
|
||||
class ProfileInheritanceTest {
|
||||
@TempDir Path temporary;
|
||||
@@ -38,18 +44,26 @@ class ProfileInheritanceTest {
|
||||
return VectorDataSettings.builder().cacheDirectory(temporary.resolve("cache")).build();
|
||||
}
|
||||
|
||||
private Path dataset(String profiles) throws IOException {
|
||||
private Path dataset(String header, String profiles) throws IOException {
|
||||
Path dir = Files.createDirectories(temporary.resolve("ds"));
|
||||
FixtureSupport.fvec(dir, "base.fvec", new float[][] {{0f, 1f}, {1f, 2f}, {2f, 3f}, {3f, 4f}});
|
||||
FixtureSupport.fvec(dir, "part.fvec", new float[][] {{7f, 7f}});
|
||||
FixtureSupport.fvec(dir, "query.fvec", new float[][] {{9f, 8f}});
|
||||
FixtureSupport.ivec(dir, "gt_default.ivecs", new int[][] {{1, 2}});
|
||||
FixtureSupport.ivec(dir, "gt_small.ivecs", new int[][] {{3, 4}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), "name: axes\nprofiles:\n" + profiles);
|
||||
FixtureSupport.ivec(dir, "results_default.ivecs", new int[][] {{0, 1, 2, 3}});
|
||||
FixtureSupport.ivec(dir, "results_small.ivecs", new int[][] {{0, 1}});
|
||||
FixtureSupport.ivec(dir, "predicates.ivecs", new int[][] {{5}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), header + "name: axes\nprofiles:\n" + profiles);
|
||||
return dir;
|
||||
}
|
||||
|
||||
private TestDataView view(Path dir, String profile) { return TestDataGroup.load(dir.toUri(), settings()).profile(profile); }
|
||||
private Path dataset(String profiles) throws IOException { return dataset("", profiles); }
|
||||
private Path layered(String profiles) throws IOException { return dataset("format_version: 3\n", profiles); }
|
||||
|
||||
private TestDataGroup group(Path dir) { return TestDataGroup.load(dir.toUri(), settings()); }
|
||||
private TestDataView view(Path dir, String profile) { return group(dir).profile(profile); }
|
||||
private VectorDataException refused(Path dir) { return assertThrows(VectorDataException.class, () -> group(dir)); }
|
||||
|
||||
@Test void aSizedProfileDoesNotInheritGroundTruthAcrossTheSizeAxis() throws Exception {
|
||||
Path dir = dataset("""
|
||||
@@ -69,12 +83,13 @@ class ProfileInheritanceTest {
|
||||
}
|
||||
|
||||
@Test void aNamedParentSharesItsOutputsAcrossAnotherAxis() throws Exception {
|
||||
Path dir = dataset("""
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
query_vectors: query.fvec
|
||||
neighbor_indices: gt_default.ivecs
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_small.ivecs
|
||||
2r_sel:
|
||||
@@ -87,6 +102,89 @@ class ProfileInheritanceTest {
|
||||
assertArrayEquals(new float[] {9f, 8f}, selective.queryVectors().get(0), "and default's facets come through the parent");
|
||||
}
|
||||
|
||||
/// The axis of a step is derived from `base_count`: a `3r` that
|
||||
/// names `2r` re-cuts the window to three and takes no ground
|
||||
/// truth, exactly as it would under `default`.
|
||||
@Test void theAxisIsDerivedFromBaseCount() throws Exception {
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
query_vectors: query.fvec
|
||||
neighbor_indices: gt_default.ivecs
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_small.ivecs
|
||||
3r:
|
||||
inherits: 2r
|
||||
base_count: 3
|
||||
""");
|
||||
TestDataView bigger = view(dir, "3r");
|
||||
assertEquals(3, bigger.baseVectors().count(), "the window is re-cut to the child's count, not the parent's");
|
||||
assertEquals("0..3", bigger.facet("base_vectors").orElseThrow().window());
|
||||
assertArrayEquals(new float[] {9f, 8f}, bigger.queryVectors().get(0));
|
||||
assertTrue(bigger.facet("neighbor_indices").isEmpty(), "ground truth does not cross a size step, whatever the parent is called");
|
||||
}
|
||||
|
||||
/// Layers stack: `default` → `2r` → a set at the same count inherits
|
||||
/// the layer's windows and unfiltered ground truth unchanged and
|
||||
/// declares its predicate group; the invariant predicate facet
|
||||
/// reaches the set through the layer.
|
||||
@Test void stackedLayersDeriveEachAxisFromBaseCount() throws Exception {
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
query_vectors: query.fvec
|
||||
metadata_predicates: predicates.ivecs
|
||||
neighbor_indices: gt_default.ivecs
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_small.ivecs
|
||||
2r-mixed:
|
||||
inherits: 2r
|
||||
base_count: 2
|
||||
metadata_results: results_small.ivecs
|
||||
2r-uniform:
|
||||
inherits: 2r
|
||||
metadata_predicates: predicates.ivecs
|
||||
metadata_results: results_small.ivecs
|
||||
""");
|
||||
TestDataView set = view(dir, "2r-mixed");
|
||||
assertEquals(2, set.baseVectors().count());
|
||||
assertArrayEquals(new int[] {3, 4}, set.neighborIndices().get(0), "the unfiltered ground truth crosses a same-size step");
|
||||
assertTrue(set.facet("metadata_predicates").isPresent(), "the invariant facet reaches the set through the layer");
|
||||
assertArrayEquals(new int[] {0, 1}, set.metadataResults().get(0));
|
||||
TestDataView uniform = view(dir, "2r-uniform");
|
||||
assertArrayEquals(new int[] {3, 4}, uniform.neighborIndices().get(0));
|
||||
assertTrue(view(dir, "2r").facet("metadata_results").isEmpty(), "a layer holds no predicate group of its own");
|
||||
}
|
||||
|
||||
/// A results index is derived from `base_count` like the ground
|
||||
/// truth: across a size step it is declared or absent, never the
|
||||
/// parent's; across a step at one size it inherits like any
|
||||
/// invariant facet.
|
||||
@Test void metadataResultsDoesNotCrossTheSizeAxis() throws Exception {
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
metadata_predicates: predicates.ivecs
|
||||
metadata_results: results_default.ivecs
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_small.ivecs
|
||||
2r-sel:
|
||||
inherits: 2r
|
||||
metadata_results: results_small.ivecs
|
||||
2r-again:
|
||||
inherits: 2r-sel
|
||||
""");
|
||||
assertTrue(view(dir, "2r").facet("metadata_results").isEmpty(), "a sized child must not take default's results index at another size");
|
||||
assertTrue(view(dir, "2r").facet("metadata_predicates").isPresent(), "the predicate facet is the same at every size and crosses");
|
||||
assertArrayEquals(new int[] {0, 1}, view(dir, "2r-again").metadataResults().get(0), "across a step at one size the results index inherits");
|
||||
}
|
||||
|
||||
@Test void aPartitionProfileInheritsNothing() throws Exception {
|
||||
Path dir = dataset("""
|
||||
default:
|
||||
@@ -101,31 +199,35 @@ class ProfileInheritanceTest {
|
||||
assertTrue(part.facet("query_vectors").isEmpty(), "and is not a windowed subset of default");
|
||||
}
|
||||
|
||||
@Test void anAlreadyWindowedBaseIsNotReWindowed() throws Exception {
|
||||
@Test void aWindowTheParentCarriesIsReCutToTheChildsCount() throws Exception {
|
||||
Path dir = dataset("""
|
||||
default:
|
||||
base_vectors: base.fvec[0..3]
|
||||
small:
|
||||
base_count: 2
|
||||
""");
|
||||
assertEquals(3, view(dir, "small").baseVectors().count(), "a window the parent carries is kept as is");
|
||||
assertEquals(2, view(dir, "small").baseVectors().count(), "the child reads the first two of the same file, not its parent's three");
|
||||
}
|
||||
|
||||
@Test void withoutABaseCountTheBaseInheritsAsIs() throws Exception {
|
||||
@Test void withoutABaseCountEveryFacetInheritsAsIs() throws Exception {
|
||||
Path dir = dataset("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
neighbor_indices: gt_default.ivecs
|
||||
variant:
|
||||
attributes: { flavour: plain }
|
||||
""");
|
||||
assertEquals(4, view(dir, "variant").baseVectors().count());
|
||||
TestDataView variant = view(dir, "variant");
|
||||
assertEquals(4, variant.baseVectors().count());
|
||||
assertArrayEquals(new int[] {1, 2}, variant.neighborIndices().get(0), "a step at the parent's size is not a size step");
|
||||
}
|
||||
|
||||
@Test void theOlderExtendsKeyStillNamesAParent() throws Exception {
|
||||
Path dir = dataset("""
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
other:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_small.ivecs
|
||||
child:
|
||||
@@ -134,25 +236,54 @@ class ProfileInheritanceTest {
|
||||
assertArrayEquals(new int[] {3, 4}, view(dir, "child").neighborIndices().get(0));
|
||||
}
|
||||
|
||||
@Test void anUnknownOrSelfParentFallsBackToDefault() throws Exception {
|
||||
Path dir = dataset("""
|
||||
@Test void aNumericRungNamesTheProfileItSpells() throws Exception {
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
query_vectors: query.fvec
|
||||
typo:
|
||||
inherits: nope
|
||||
selfish:
|
||||
inherits: selfish
|
||||
100:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_small.ivecs
|
||||
child:
|
||||
inherits: 100
|
||||
""");
|
||||
assertArrayEquals(new float[] {9f, 8f}, view(dir, "typo").queryVectors().get(0));
|
||||
assertArrayEquals(new float[] {9f, 8f}, view(dir, "selfish").queryVectors().get(0));
|
||||
assertArrayEquals(new int[] {3, 4}, view(dir, "child").neighborIndices().get(0), "a parent YAML read as a number is the name it spells");
|
||||
}
|
||||
|
||||
@Test void aCycleLeavesItsMembersWithWhatTheyDeclare() throws Exception {
|
||||
Path dir = dataset("""
|
||||
// -- format_version 3: stated parents --
|
||||
|
||||
@Test void versionThreeRefusesAnUnstatedParent() throws Exception {
|
||||
VectorDataException e = refused(layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
small:
|
||||
base_count: 2
|
||||
"""));
|
||||
assertTrue(e.getMessage().contains("`small` names no parent") && e.getMessage().contains("inherits: default"), e.getMessage());
|
||||
}
|
||||
|
||||
@Test void versionThreeRefusesAnUnknownOrSelfParent() throws Exception {
|
||||
VectorDataException unknown = refused(layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
child:
|
||||
inherits: nonexistent
|
||||
query_vectors: query.fvec
|
||||
"""));
|
||||
assertTrue(unknown.getMessage().contains("`child`") && unknown.getMessage().contains("unknown parent `nonexistent`"), unknown.getMessage());
|
||||
VectorDataException selfish = refused(layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
a:
|
||||
inherits: a
|
||||
"""));
|
||||
assertTrue(selfish.getMessage().contains("`a` names itself"), selfish.getMessage());
|
||||
}
|
||||
|
||||
@Test void versionThreeRefusesACycleNamingItsMembers() throws Exception {
|
||||
VectorDataException e = refused(layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
a:
|
||||
inherits: b
|
||||
base_vectors: part.fvec
|
||||
@@ -161,9 +292,75 @@ class ProfileInheritanceTest {
|
||||
query_vectors: query.fvec
|
||||
c:
|
||||
inherits: a
|
||||
"""));
|
||||
assertTrue(e.getMessage().contains("`a`, `b` close an inheritance cycle"), e.getMessage());
|
||||
assertEquals(1, e.getMessage().split("cycle", -1).length - 1, "one cycle, reported once: " + e.getMessage());
|
||||
}
|
||||
|
||||
@Test void versionThreeRefusesAPartitionThatNamesAParentAndAcceptsOneAlone() throws Exception {
|
||||
VectorDataException both = refused(layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
p:
|
||||
partition: true
|
||||
inherits: default
|
||||
base_vectors: part.fvec
|
||||
"""));
|
||||
assertTrue(both.getMessage().contains("both `partition: true` and `inherits:`"), both.getMessage());
|
||||
Path alone = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
p:
|
||||
partition: true
|
||||
base_vectors: part.fvec
|
||||
""");
|
||||
assertTrue(view(dir, "a").facet("query_vectors").isEmpty(), "a cycle never settles, so nothing is inherited through it");
|
||||
assertTrue(view(dir, "b").facet("base_vectors").isEmpty());
|
||||
assertTrue(view(dir, "c").facet("base_vectors").isEmpty(), "nor by a profile whose parent is in one");
|
||||
assertEquals(1, view(alone, "p").baseVectors().count());
|
||||
}
|
||||
|
||||
@Test void aNamedParentIsHeldToVersionOneWhenUnversioned() throws Exception {
|
||||
VectorDataException e = refused(dataset("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
2r:
|
||||
base_count: 2
|
||||
2r_sel:
|
||||
inherits: 2r
|
||||
query_vectors: query.fvec
|
||||
"""));
|
||||
assertTrue(e.getMessage().contains("no format_version") && e.getMessage().contains("format_version: 3"), e.getMessage());
|
||||
VectorDataException understated = refused(dataset("format_version: 2\n", """
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
2r:
|
||||
base_count: 2
|
||||
2r_sel:
|
||||
inherits: 2r
|
||||
"""));
|
||||
assertTrue(understated.getMessage().contains("understate"), understated.getMessage());
|
||||
assertNotNull(group(dataset("format_version: 2\n", """
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
""")), "a parent of default needs nothing above 1");
|
||||
}
|
||||
|
||||
// -- format_version 3: the tag schema --
|
||||
|
||||
@Test void aTagSchemaLoadsInNamingOrderAndNeedsVersionThree() throws Exception {
|
||||
Path dir = dataset("format_version: 3\nprofile_tags:\n size: ~\n predicates: ~\n selectivity: ~\n family: stratified\n", """
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
""");
|
||||
Map<String, Object> tags = group(dir).profileTags();
|
||||
assertEquals(List.of("size", "predicates", "selectivity", "family"), List.copyOf(tags.keySet()), "the schema loads in naming order");
|
||||
assertTrue(tags.containsKey("size") && tags.get("size") == null, "a naming tag carries no default");
|
||||
assertEquals("stratified", tags.get("family"));
|
||||
assertEquals(3, group(dir).formatVersion());
|
||||
VectorDataException unversioned = refused(dataset("profile_tags:\n size: ~\n", " default:\n base_vectors: base.fvec\n"));
|
||||
assertTrue(unversioned.getMessage().contains("format_version: 3"), "a schema alone needs version 3: " + unversioned.getMessage());
|
||||
assertTrue(refused(dataset("format_version: 2\nprofile_tags:\n size: ~\n", " default:\n base_vectors: base.fvec\n")).getMessage().contains("understate"));
|
||||
assertTrue(group(dataset(" default:\n base_vectors: base.fvec\n")).profileTags().isEmpty(), "a dataset without a schema declares none");
|
||||
}
|
||||
}
|
||||
@@ -261,6 +261,7 @@ class RecordBindingTest {
|
||||
}
|
||||
FixtureSupport.fvec(dir, "base.fvec", new float[][] {{0f, 1f, 2f, 3f}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: sh
|
||||
profiles:
|
||||
default:
|
||||
|
||||
@@ -75,6 +75,7 @@ class RecordFacetsTest {
|
||||
FixtureSupport.slabOf(dir, "meta__0002.slab", metadataRecords(20, 5), 4);
|
||||
FixtureSupport.fvec(dir, "base.fvec", new float[][] {{0f, 1f, 2f, 3f}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: sharded-meta
|
||||
profiles:
|
||||
default:
|
||||
|
||||
@@ -61,6 +61,7 @@ class ShardedFacetsTest {
|
||||
fvec(dir, "base__0001.fvec", 100, 100);
|
||||
fvec(dir, "base__0002.fvec", 40, 200);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: sharded
|
||||
profiles:
|
||||
default:
|
||||
@@ -106,6 +107,7 @@ class ShardedFacetsTest {
|
||||
Path dir = Files.createDirectories(temporary.resolve("bad"));
|
||||
fvec(dir, "base__0000.fvec", 100, 0);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: bad
|
||||
profiles:
|
||||
default:
|
||||
@@ -122,6 +124,7 @@ class ShardedFacetsTest {
|
||||
@Test void anExplicitSeriesOpensFromNamedFiles() throws Exception {
|
||||
Path dir = uniformSeries("");
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: explicit
|
||||
profiles:
|
||||
default:
|
||||
@@ -140,6 +143,7 @@ class ShardedFacetsTest {
|
||||
@Test void bareNamesResolveToTheSameFacetAsCountedOnes() throws Exception {
|
||||
Path dir = uniformSeries("");
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: bare
|
||||
profiles:
|
||||
default:
|
||||
@@ -158,6 +162,7 @@ class ShardedFacetsTest {
|
||||
for (int i = 0; i < values.length; i++) values[i] = i;
|
||||
FixtureSupport.scalarI32(dir, "corpus.u32", values);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: sliced
|
||||
profiles:
|
||||
default:
|
||||
@@ -189,6 +194,7 @@ class ShardedFacetsTest {
|
||||
Path dir = Files.createDirectories(temporary.resolve("gap"));
|
||||
fvec(dir, "base__0000.fvec", 100, 0);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: gap
|
||||
profiles:
|
||||
default:
|
||||
@@ -211,6 +217,7 @@ class ShardedFacetsTest {
|
||||
fvec(dir, "base__0000.fvec", 100, 0);
|
||||
fvec(dir, "base__0001.fvec", 100, 100);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: win
|
||||
profiles:
|
||||
default:
|
||||
@@ -235,6 +242,7 @@ class ShardedFacetsTest {
|
||||
@Test void aWindowOnAUniformPatternBoundsTheReader() throws Exception {
|
||||
Path dir = uniformSeries("");
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: suffix
|
||||
profiles:
|
||||
default:
|
||||
@@ -249,6 +257,7 @@ class ShardedFacetsTest {
|
||||
assertEquals(149 * 100f, base.get(149)[0], "record 149 lives in shard 1");
|
||||
assertEquals("0..150", view(dir).facet("base_vectors").orElseThrow().window(), "the suffix is reported as the facet window");
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: twice
|
||||
profiles:
|
||||
default:
|
||||
@@ -283,6 +292,7 @@ class ShardedFacetsTest {
|
||||
fvec(dir, "base__0000.fvec", 10, 0);
|
||||
fvec(dir, "base__0001.fvec", 10, 10);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: nm
|
||||
profiles:
|
||||
default:
|
||||
@@ -315,6 +325,7 @@ class ShardedFacetsTest {
|
||||
FixtureSupport.vvec(dir, "meta__0001.ivvec", second);
|
||||
fvec(dir, "base.fvec", 8, 0);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: vvec
|
||||
profiles:
|
||||
default:
|
||||
@@ -419,14 +430,33 @@ class ShardedFacetsTest {
|
||||
assertTrue(refused.getMessage().contains("understate"), refused.getMessage());
|
||||
}
|
||||
|
||||
@Test void anAbsentVersionIsNotAnUnderstatement() throws Exception {
|
||||
TestDataView view = view(uniformSeries(""));
|
||||
assertEquals(240, view.baseVectors().count(), "an unannotated sharded dataset loads: absence is not a claim");
|
||||
Path generous = uniformSeries("");
|
||||
Files.writeString(generous.resolve("dataset.yaml"), "format_version: 2\n" + Files.readString(generous.resolve("dataset.yaml")));
|
||||
assertEquals(240, view(generous).baseVectors().count(), "a version equal to the content's requirement is accepted");
|
||||
@Test void anAbsentVersionIsHeldToOne() throws Exception {
|
||||
// The uniform fixture states version 2; without it the same
|
||||
// content is refused naming the version to declare, because an
|
||||
// unversioned manifest is read as the least it could be.
|
||||
Path unversioned = uniformSeries("");
|
||||
Files.writeString(unversioned.resolve("dataset.yaml"), Files.readString(unversioned.resolve("dataset.yaml")).replace("format_version: 2\n", ""));
|
||||
VectorDataException refused = assertThrows(VectorDataException.class, () -> TestDataGroup.load(unversioned.toUri(), settings()),
|
||||
"an unannotated sharded dataset is held to version 1 and refused");
|
||||
assertTrue(refused.getMessage().contains("no format_version"), refused.getMessage());
|
||||
assertTrue(refused.getMessage().contains("format_version: 2"), "names the version to declare: " + refused.getMessage());
|
||||
assertEquals(240, view(uniformSeries("")).baseVectors().count(), "a version equal to the content's requirement is accepted");
|
||||
Path plain = dataset("generous", "format_version: 2\nname: g\nprofiles:\n default:\n base_vectors: base.fvec\n");
|
||||
assertNotNull(TestDataGroup.load(plain.toUri(), settings()), "a version higher than the content requires is merely generous");
|
||||
assertEquals(2, TestDataGroup.load(plain.toUri(), settings()).formatVersion(), "the stated version is exposed");
|
||||
assertEquals(1, TestDataGroup.load(dataset("plainest", "name: p\nprofiles:\n default:\n base_vectors: base.fvec\n").toUri(), settings()).formatVersion(),
|
||||
"an absent version reads as 1");
|
||||
}
|
||||
|
||||
@Test void aSeriesReportsItsShardCount() throws Exception {
|
||||
TestDataView view = view(uniformSeries(""));
|
||||
assertEquals(3, view.facet("base_vectors").orElseThrow().shardCount(), "a uniform series says how many files it spans");
|
||||
Path dir = Files.createDirectories(temporary.resolve("explicit-count"));
|
||||
fvec(dir, "a.fvec", 2, 0); fvec(dir, "b.fvec", 2, 2);
|
||||
TestDataView explicit = view(dataset("explicit-count", "format_version: 2\nname: e\nprofiles:\n default:\n base_vectors:\n source: [ a.fvec=2, b.fvec=2 ]\n record_count: 4\n"));
|
||||
assertNull(explicit.facet("base_vectors").orElseThrow().shardCount(), "the explicit form's files are its entries");
|
||||
assertEquals(2, explicit.facet("base_vectors").orElseThrow().series().entries().size());
|
||||
assertNull(view(dataset("one", "name: one\nprofiles:\n default:\n base_vectors: base.fvec\n")).facet("base_vectors").orElseThrow().shardCount(), "a single file spans none");
|
||||
}
|
||||
|
||||
// -- Files that are not one facet, and what a series promises --
|
||||
@@ -436,6 +466,7 @@ class ShardedFacetsTest {
|
||||
fvec(dir, "base__0000.fvec", 10, 0);
|
||||
FixtureSupport.fvec(dir, "base__0001.fvec", new float[10][8]);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: disagree
|
||||
profiles:
|
||||
default:
|
||||
@@ -456,6 +487,7 @@ class ShardedFacetsTest {
|
||||
fvec(dir, "base__0001.fvec", 10, 10);
|
||||
fvec(dir, "base__0003.fvec", 5, 30);
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: midgap
|
||||
profiles:
|
||||
default:
|
||||
|
||||
@@ -139,6 +139,7 @@ class SlabWindowsTest {
|
||||
FixtureSupport.slab(dir, "m__0001.slab", 50, 25, 64);
|
||||
FixtureSupport.fvec(dir, "base.fvec", new float[][] {{0f, 1f, 2f, 3f}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: slab-series
|
||||
profiles:
|
||||
default:
|
||||
@@ -171,6 +172,7 @@ class SlabWindowsTest {
|
||||
FixtureSupport.slab(dir, "m__0001.slab", 20, 10, 64);
|
||||
FixtureSupport.fvec(dir, "base.fvec", new float[][] {{0f, 1f, 2f, 3f}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: visit-sharded
|
||||
profiles:
|
||||
default:
|
||||
@@ -194,6 +196,7 @@ class SlabWindowsTest {
|
||||
FixtureSupport.slab(dir, "m.slab", 200, 25, 64);
|
||||
FixtureSupport.fvec(dir, "base.fvec", new float[][] {{0f, 1f, 2f, 3f}});
|
||||
Files.writeString(dir.resolve("dataset.yaml"), """
|
||||
format_version: 2
|
||||
name: sliced-slab
|
||||
profiles:
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright (c) 2026 The NoSQLBench Authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.nosqlbench.vectordata.internal;
|
||||
|
||||
import io.nosqlbench.vectordata.VectorDataException;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
@Tag("unit")
|
||||
class ProfileParentsTest {
|
||||
private static ProfileParents.Facts f(String name, boolean partition, String inherits) { return new ProfileParents.Facts(name, partition, inherits); }
|
||||
|
||||
/// Version 3 refuses an unstated, unknown, self or cyclic parent and
|
||||
/// a partition that names one; a partition alone is accepted, and
|
||||
/// the same shapes pass below 3.
|
||||
@Test void versionThreeStatesEveryParent() {
|
||||
List<ProfileParents.Facts> ok = List.of(f("default", false, null), f("10m", false, "default"), f("part", true, null), f("set", false, "10m"));
|
||||
assertDoesNotThrow(() -> ProfileParents.check(3, ok));
|
||||
|
||||
List<ProfileParents.Facts> unstated = List.of(f("default", false, null), f("10m-mixed", false, null));
|
||||
String e = assertThrows(VectorDataException.class, () -> ProfileParents.check(3, unstated)).getMessage();
|
||||
assertTrue(e.contains("`10m-mixed` names no parent") && e.contains("inherits: default"), e);
|
||||
assertDoesNotThrow(() -> ProfileParents.check(2, unstated), "below 3 the fallback stands");
|
||||
|
||||
String both = assertThrows(VectorDataException.class, () -> ProfileParents.check(3, List.of(f("default", false, null), f("p", true, "default")))).getMessage();
|
||||
assertTrue(both.contains("both `partition: true` and `inherits:`"), both);
|
||||
String unknown = assertThrows(VectorDataException.class, () -> ProfileParents.check(3, List.of(f("default", false, null), f("a", false, "nope")))).getMessage();
|
||||
assertTrue(unknown.contains("unknown parent `nope`"), unknown);
|
||||
String selfish = assertThrows(VectorDataException.class, () -> ProfileParents.check(3, List.of(f("default", false, null), f("a", false, "a")))).getMessage();
|
||||
assertTrue(selfish.contains("names itself"), selfish);
|
||||
String cyclic = assertThrows(VectorDataException.class, () -> ProfileParents.check(3, List.of(f("default", false, null), f("a", false, "b"), f("b", false, "a")))).getMessage();
|
||||
assertTrue(cyclic.contains("`a`, `b` close an inheritance cycle"), cyclic);
|
||||
assertEquals(1, cyclic.split("cycle", -1).length - 1, "one cycle, reported once: " + cyclic);
|
||||
String base = assertThrows(VectorDataException.class, () -> ProfileParents.check(3, List.of(f("default", false, "x")))).getMessage();
|
||||
assertTrue(base.contains("`default` is the base layer"), base);
|
||||
}
|
||||
|
||||
/// Below 3 the conditions are advisories naming the version that
|
||||
/// will refuse them; at 3 there are none, the loader refuses.
|
||||
@Test void advisoriesPrecedeTheRefusal() {
|
||||
List<ProfileParents.Facts> rows = List.of(f("default", false, null), f("10m", false, null), f("a", false, "nope"));
|
||||
List<String> notes = ProfileParents.advisories(2, rows);
|
||||
assertEquals(2, notes.size(), notes.toString());
|
||||
assertTrue(notes.stream().allMatch(n -> n.endsWith("(refused from format_version 3)")), notes.toString());
|
||||
assertTrue(ProfileParents.advisories(3, rows).isEmpty());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user