mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2026-10-12 09:24:52 -05:00
fix(vectordata): resolve sized profiles against the default profile
Published sized profiles name only what differs: a windowed base
override and their own neighbor facets, with no extends key. Facets a
profile does not declare — query vectors included — resolve from
'default' implicitly, matching the expanded layout the publishing
toolchain emits into catalogs. Java honored only explicit extends, so
a released dataset failed with 'Profile 100k lacks facet
query_vectors' while its own catalog showed the facet plainly.
The window: key also accepts the serializer's structured form (a list
of {min_incl, max_excl} maps) and bare counts; dropping an
unrecognized form silently would read the whole facet where a window
was declared, so it fails instead.
Verified against the real remote dataset: profile 100k resolves all
four facets, reads query vectors, and clips the base to its window.
This commit is contained in:
1 parent
33cbde93fb
commit
5b992e89df
3 files changed
+91
-4
No files matched your search
@@ -21,9 +21,16 @@ implementation's current behavior and its format tests are normative.
|
||||
- Windows: the full `DSWindow` grammar — `..` bounds, open ends, `(`/`]`
|
||||
bound adjustment, count/size suffixes (`K`/`M`/`G`/`T`, `KB`…`TiB`,
|
||||
compound `1g24m`), comma-separated intervals, and the structural
|
||||
window-suffix sugar on source strings (`base.fvec[0..1M)`). Intervals
|
||||
that select no records are rejected at parse. Readers apply the first
|
||||
interval and clamp both bounds to the data.
|
||||
window-suffix sugar on source strings (`base.fvec[0..1M)`). The
|
||||
serialized structured form — a list of `{min_incl, max_excl}` maps —
|
||||
and bare counts (`[0..N)`) are accepted on the `window:` key.
|
||||
Intervals that select no records are rejected at parse. Readers apply
|
||||
the first interval and clamp both bounds to the data.
|
||||
- Profiles: facets a profile does not declare resolve from `default`
|
||||
implicitly, before any explicit `extends` — published sized profiles
|
||||
(`100k`, `1m`, ...) carry a windowed base and their own neighbor
|
||||
facets and inherit the rest, matching the expanded layout the
|
||||
publishing toolchain emits.
|
||||
- Prefetch: caller-supplied record windows on any facet via
|
||||
`prefetchPlan`, `prefetch`, and `prefetchInBackground` on
|
||||
`TestDataView`, with `WholeFacetFallback` consent gating, chunk-level
|
||||
|
||||
@@ -23,7 +23,9 @@ import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@@ -98,6 +100,12 @@ public final class TestDataGroup {
|
||||
private Map<String, FacetDescriptor> facets(String profile, Map<String, FacetDescriptor> inherited) {
|
||||
Map<String, Object> definition = profiles.get(profile);
|
||||
if (definition == null) throw new VectorDataException("Dataset " + name + " has no profile " + profile);
|
||||
// Profiles name only what differs: facets a profile does not
|
||||
// declare resolve from `default`. A sized profile like `100k`
|
||||
// carries a windowed base and its own neighbor facets and
|
||||
// inherits the rest — query vectors included — without an
|
||||
// explicit `extends`.
|
||||
if (!"default".equals(profile) && profiles.containsKey("default")) inherited.putAll(facets("default", new LinkedHashMap<>()));
|
||||
String parent = YamlData.optionalString(definition.get("extends"));
|
||||
if (parent != null) {
|
||||
if (parent.equals(profile)) throw new VectorDataException("Profile cannot extend itself: " + profile);
|
||||
@@ -110,13 +118,41 @@ public final class TestDataGroup {
|
||||
String source = YamlData.optionalString(values.get("source"));
|
||||
if (source == null) source = YamlData.optionalString(values.get("path"));
|
||||
if (source == null) continue;
|
||||
String window = YamlData.optionalString(values.get("window"));
|
||||
String window = windowText(values.get("window"));
|
||||
String[] split = splitWindowSuffix(source);
|
||||
if (split[1] != null) window = split[1];
|
||||
inherited.put(facetName, new FacetDescriptor(facetName, manifest.resolve(split[0]), window, Map.copyOf(values)));
|
||||
}
|
||||
return inherited;
|
||||
}
|
||||
/// Normalizes the `window:` key to the canonical interval text.
|
||||
/// Manifests carry it as the string grammar, but the serializer
|
||||
/// that publishes profiles emits the structured form — a list of
|
||||
/// `{min_incl, max_excl}` maps — and a bare count means `[0..N)`.
|
||||
/// Dropping an unrecognized form silently would read the whole
|
||||
/// facet where a window was declared, so it fails instead.
|
||||
private static String windowText(Object value) {
|
||||
if (value == null) return null;
|
||||
if (value instanceof String text) return text;
|
||||
if (value instanceof Number count) return "0.." + count;
|
||||
if (value instanceof List<?> list) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
for (Object item : list) parts.add(intervalText(item));
|
||||
return String.join(", ", parts);
|
||||
}
|
||||
if (value instanceof Map<?, ?>) return intervalText(value);
|
||||
throw new VectorDataException("Unrecognized window form: " + value);
|
||||
}
|
||||
private static String intervalText(Object item) {
|
||||
if (item instanceof String text) return text;
|
||||
if (item instanceof Number count) return "0.." + count;
|
||||
if (item instanceof Map<?, ?> raw) {
|
||||
Map<String, Object> interval = YamlData.map(raw, "window interval");
|
||||
Object min = interval.get("min_incl"); Object max = interval.get("max_excl");
|
||||
if (min instanceof Number && max instanceof Number) return min + ".." + max;
|
||||
}
|
||||
throw new VectorDataException("Unrecognized window interval: " + item);
|
||||
}
|
||||
/// Splits the documented window-suffix sugar off a facet source
|
||||
/// string — `base.fvec[0..1M)` names the file plus a record window,
|
||||
/// with either bracket kind on either side. The outer delimiters
|
||||
|
||||
@@ -87,6 +87,50 @@ class CatalogManifestTest {
|
||||
assertTrue(view.facet("base").isPresent(), "shorthand aliases resolve on lookup");
|
||||
assertTrue(view.facet("train").isPresent(), "every Rust alias resolves");
|
||||
}
|
||||
@Test void sizedProfilesInheritMissingFacetsFromDefault() throws Exception {
|
||||
// Published sized profiles name only what differs: a windowed
|
||||
// base override and their own neighbor facets, with no
|
||||
// `extends`. Everything else — query vectors included —
|
||||
// resolves from `default`, matching the expanded layout the
|
||||
// publishing toolchain emits. A released dataset failed here
|
||||
// with "Profile 100k lacks facet query_vectors".
|
||||
Path dataset = Files.createDirectories(temporary.resolve("sized"));
|
||||
FixtureSupport.fvec(dataset, "base.fvec", new float[][] {{0f, 1f}, {1f, 2f}, {2f, 3f}, {3f, 4f}});
|
||||
FixtureSupport.fvec(dataset, "query.fvec", new float[][] {{9f, 8f}});
|
||||
FixtureSupport.ivec(dataset, "gt_default.ivecs", new int[][] {{1, 2}});
|
||||
FixtureSupport.ivec(dataset, "gt_small.ivecs", new int[][] {{3, 4}});
|
||||
Files.writeString(dataset.resolve("dataset.yaml"), """
|
||||
name: sized
|
||||
profiles:
|
||||
default:
|
||||
maxk: 100
|
||||
base_vectors: base.fvec
|
||||
query_vectors: query.fvec
|
||||
neighbor_indices: gt_default.ivecs
|
||||
small:
|
||||
maxk: 100
|
||||
base_count: 2
|
||||
base_vectors:
|
||||
source: base.fvec
|
||||
window: "[0..2]"
|
||||
neighbor_indices: gt_small.ivecs
|
||||
structured:
|
||||
base_vectors:
|
||||
source: base.fvec
|
||||
window:
|
||||
- min_incl: 1
|
||||
max_excl: 3
|
||||
""");
|
||||
TestDataView small = TestDataGroup.load(dataset.toUri(), settings()).profile("small");
|
||||
assertArrayEquals(new float[] {9f, 8f}, small.queryVectors().get(0), "missing facets inherit from default");
|
||||
assertEquals(2, small.baseVectors().count(), "the sized window clips the shared base");
|
||||
assertArrayEquals(new int[] {3, 4}, small.neighborIndices().get(0), "declared facets override inherited ones");
|
||||
// The serializer's structured window form clips identically to
|
||||
// the string grammar.
|
||||
TestDataView structured = TestDataGroup.load(dataset.toUri(), settings()).profile("structured");
|
||||
assertEquals(2, structured.baseVectors().count());
|
||||
assertArrayEquals(new float[] {1f, 2f}, structured.baseVectors().get(0));
|
||||
}
|
||||
@Test void opensLegacyKnnEntriesProfilesWithoutDatasetManifest() throws Exception {
|
||||
FixtureSupport.fvec(temporary, "base.fvec", new float[][] {{1f, 2f}});
|
||||
FixtureSupport.fvec(temporary, "query.fvec", new float[][] {{3f, 4f}});
|
||||
|
||||
Reference in new issue
Block a user