fix(vectordata): resolve sized profiles against the default profile

Published sized profiles name only what differs: a windowed base
override and their own neighbor facets, with no extends key. Facets a
profile does not declare — query vectors included — resolve from
'default' implicitly, matching the expanded layout the publishing
toolchain emits into catalogs. Java honored only explicit extends, so
a released dataset failed with 'Profile 100k lacks facet
query_vectors' while its own catalog showed the facet plainly.

The window: key also accepts the serializer's structured form (a list
of {min_incl, max_excl} maps) and bare counts; dropping an
unrecognized form silently would read the whole facet where a window
was declared, so it fails instead.

Verified against the real remote dataset: profile 100k resolves all
four facets, reads query vectors, and clips the base to its window.
This commit is contained in:
Jonathan Shook committed 2026-08-26 20:30:19 +00:00
1 parent 33cbde93fb
commit 5b992e89df
3 files changed
+91 -4

No files matched your search

+10 -3
View File
@@ -21,9 +21,16 @@ implementation's current behavior and its format tests are normative.
- Windows: the full `DSWindow` grammar — `..` bounds, open ends, `(`/`]`
bound adjustment, count/size suffixes (`K`/`M`/`G`/`T`, `KB`…`TiB`,
compound `1g24m`), comma-separated intervals, and the structural
window-suffix sugar on source strings (`base.fvec[0..1M)`). Intervals
that select no records are rejected at parse. Readers apply the first
interval and clamp both bounds to the data.
window-suffix sugar on source strings (`base.fvec[0..1M)`). The
serialized structured form — a list of `{min_incl, max_excl}` maps —
and bare counts (`[0..N)`) are accepted on the `window:` key.
Intervals that select no records are rejected at parse. Readers apply
the first interval and clamp both bounds to the data.
- Profiles: facets a profile does not declare resolve from `default`
implicitly, before any explicit `extends` — published sized profiles
(`100k`, `1m`, ...) carry a windowed base and their own neighbor
facets and inherit the rest, matching the expanded layout the
publishing toolchain emits.
- Prefetch: caller-supplied record windows on any facet via
`prefetchPlan`, `prefetch`, and `prefetchInBackground` on
`TestDataView`, with `WholeFacetFallback` consent gating, chunk-level
@@ -23,7 +23,9 @@ import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -98,6 +100,12 @@ public final class TestDataGroup {
private Map<String, FacetDescriptor> facets(String profile, Map<String, FacetDescriptor> inherited) {
Map<String, Object> definition = profiles.get(profile);
if (definition == null) throw new VectorDataException("Dataset " + name + " has no profile " + profile);
// Profiles name only what differs: facets a profile does not
// declare resolve from `default`. A sized profile like `100k`
// carries a windowed base and its own neighbor facets and
// inherits the rest — query vectors included — without an
// explicit `extends`.
if (!"default".equals(profile) && profiles.containsKey("default")) inherited.putAll(facets("default", new LinkedHashMap<>()));
String parent = YamlData.optionalString(definition.get("extends"));
if (parent != null) {
if (parent.equals(profile)) throw new VectorDataException("Profile cannot extend itself: " + profile);
@@ -110,13 +118,41 @@ public final class TestDataGroup {
String source = YamlData.optionalString(values.get("source"));
if (source == null) source = YamlData.optionalString(values.get("path"));
if (source == null) continue;
String window = YamlData.optionalString(values.get("window"));
String window = windowText(values.get("window"));
String[] split = splitWindowSuffix(source);
if (split[1] != null) window = split[1];
inherited.put(facetName, new FacetDescriptor(facetName, manifest.resolve(split[0]), window, Map.copyOf(values)));
}
return inherited;
}
/// Normalizes the `window:` key to the canonical interval text.
/// Manifests carry it as the string grammar, but the serializer
/// that publishes profiles emits the structured form — a list of
/// `{min_incl, max_excl}` maps — and a bare count means `[0..N)`.
/// Dropping an unrecognized form silently would read the whole
/// facet where a window was declared, so it fails instead.
private static String windowText(Object value) {
if (value == null) return null;
if (value instanceof String text) return text;
if (value instanceof Number count) return "0.." + count;
if (value instanceof List<?> list) {
List<String> parts = new ArrayList<>();
for (Object item : list) parts.add(intervalText(item));
return String.join(", ", parts);
}
if (value instanceof Map<?, ?>) return intervalText(value);
throw new VectorDataException("Unrecognized window form: " + value);
}
private static String intervalText(Object item) {
if (item instanceof String text) return text;
if (item instanceof Number count) return "0.." + count;
if (item instanceof Map<?, ?> raw) {
Map<String, Object> interval = YamlData.map(raw, "window interval");
Object min = interval.get("min_incl"); Object max = interval.get("max_excl");
if (min instanceof Number && max instanceof Number) return min + ".." + max;
}
throw new VectorDataException("Unrecognized window interval: " + item);
}
/// Splits the documented window-suffix sugar off a facet source
/// string — `base.fvec[0..1M)` names the file plus a record window,
/// with either bracket kind on either side. The outer delimiters
@@ -87,6 +87,50 @@ class CatalogManifestTest {
assertTrue(view.facet("base").isPresent(), "shorthand aliases resolve on lookup");
assertTrue(view.facet("train").isPresent(), "every Rust alias resolves");
}
@Test void sizedProfilesInheritMissingFacetsFromDefault() throws Exception {
// Published sized profiles name only what differs: a windowed
// base override and their own neighbor facets, with no
// `extends`. Everything else — query vectors included —
// resolves from `default`, matching the expanded layout the
// publishing toolchain emits. A released dataset failed here
// with "Profile 100k lacks facet query_vectors".
Path dataset = Files.createDirectories(temporary.resolve("sized"));
FixtureSupport.fvec(dataset, "base.fvec", new float[][] {{0f, 1f}, {1f, 2f}, {2f, 3f}, {3f, 4f}});
FixtureSupport.fvec(dataset, "query.fvec", new float[][] {{9f, 8f}});
FixtureSupport.ivec(dataset, "gt_default.ivecs", new int[][] {{1, 2}});
FixtureSupport.ivec(dataset, "gt_small.ivecs", new int[][] {{3, 4}});
Files.writeString(dataset.resolve("dataset.yaml"), """
name: sized
profiles:
default:
maxk: 100
base_vectors: base.fvec
query_vectors: query.fvec
neighbor_indices: gt_default.ivecs
small:
maxk: 100
base_count: 2
base_vectors:
source: base.fvec
window: "[0..2]"
neighbor_indices: gt_small.ivecs
structured:
base_vectors:
source: base.fvec
window:
- min_incl: 1
max_excl: 3
""");
TestDataView small = TestDataGroup.load(dataset.toUri(), settings()).profile("small");
assertArrayEquals(new float[] {9f, 8f}, small.queryVectors().get(0), "missing facets inherit from default");
assertEquals(2, small.baseVectors().count(), "the sized window clips the shared base");
assertArrayEquals(new int[] {3, 4}, small.neighborIndices().get(0), "declared facets override inherited ones");
// The serializer's structured window form clips identically to
// the string grammar.
TestDataView structured = TestDataGroup.load(dataset.toUri(), settings()).profile("structured");
assertEquals(2, structured.baseVectors().count());
assertArrayEquals(new float[] {1f, 2f}, structured.baseVectors().get(0));
}
@Test void opensLegacyKnnEntriesProfilesWithoutDatasetManifest() throws Exception {
FixtureSupport.fvec(temporary, "base.fvec", new float[][] {{1f, 2f}});
FixtureSupport.fvec(temporary, "query.fvec", new float[][] {{3f, 4f}});