diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index 4854da000..75056e352 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -296,10 +296,17 @@ java -jar nb5.jar run driver=stdout cycles=10 \ ``` Prefetch modes on every mapper: `eager` (default; aliases `prebuffer`, -`true`), `background`, `none` (aliases `demand`, `false`). A binding -window is in the reader's own coordinates — after any profile window — -and is shifted to absolute records for the fetch, so the warmed bytes -are exactly the bytes the clipped reader exposes. +`true`), `background`, `none` (aliases `demand`, `false`). + +A binding window says **which records to warm**, and nothing else — +indices stay absolute record ordinals. +`BaseVectors('ds:profile','[50000..100000)')` warms records 50000 +through 99999, and a run of `cycles=50000..100000` addresses exactly +those records. Reading outside the window is not an error; it just +demand-pages, as it would with no window. Ordinals mean the same thing +here, in `VariableFacet`, and in `prefetchCycles`. The window is +translated into dataset coordinates for the fetch, so the warmed bytes +are the bytes the reader will expose. ## Verifying that warming happened diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java index b8b16a94f..df65874b6 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java @@ -126,7 +126,7 @@ public class VectorDataExprs implements ExprFunctionProvider { @ExprFunctionSpec( name = "windowedFacet", synopsis = "windowedFacet(\"dataset:profile\", \"facet_name\", \"window\")", - description = "Return a reader for a named facet, clipped to a record window in the reader's own coordinates." + description = "Return a reader for a named facet, restricted to a record window. Indices stay absolute — a window of [50000..100000) accepts 50000..99999 — so cycle values address the records they name." ) public VectorReader windowedFacet(String datasetNameAndProfile, String facetName, String window) { return WindowedReader.clip(dataset(datasetNameAndProfile).openFacet(facetName), window); diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java index 53ebd824b..090d9ef99 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java @@ -35,9 +35,8 @@ import java.util.Locale; import java.util.function.LongFunction; /// Base plumbing for vectordata-backed binding functions: opens the -/// named dataset profile from the default catalogs, clips the reader to -/// an optional record window, and warms data according to a prefetch -/// mode: +/// named dataset profile from the default catalogs and warms an +/// optional record window according to a prefetch mode: /// /// - `eager` (aliases `prebuffer`, `true`) — fetch the effective window, /// or the whole facet when no window applies, before returning; @@ -45,11 +44,18 @@ import java.util.function.LongFunction; /// immediately, letting reads overlap the download; /// - `none` (aliases `demand`, `false`) — demand-paged access only. /// -/// A binding window is expressed in the reader's own index coordinates, -/// after any profile-declared window. For warming, it is translated to -/// absolute record coordinates by shifting against the profile window, -/// so the bytes fetched are exactly the bytes the clipped reader -/// exposes. +/// A binding window selects **which records to warm**, and nothing +/// else: indices stay absolute record ordinals, so a workload running +/// `cycles=50000..100000` with a window of `[50000..100000)` addresses +/// the records those cycles name. Reads outside the window are not an +/// error — they simply demand-page, the way they would with no window +/// at all — which also keeps the binding resolvable, since VirtData +/// probes a newly resolved function with a small sample index before +/// any cycle runs. Ordinals mean the same thing here, in +/// [VariableFacet], and in the `prefetchCycles` expression function. +/// The window is translated into dataset coordinates against any +/// profile-declared window, so the bytes fetched are the bytes the +/// reader will expose. public abstract class CoreVectors implements LongFunction { protected final TestDataView tdv; @@ -66,7 +72,8 @@ public abstract class CoreVectors implements LongFunction { Catalog catalog = Catalog.of(CatalogSources.defaults(), settings); tdv = catalog.openProfile(datasetAndProfile); this.facetName = FacetNames.canonical(facetName); - dataset = WindowedReader.clip(getRandomAccessData(), window); + // Deliberately unwrapped: the window warms, it does not clip. + dataset = getRandomAccessData(); backgroundPrefetch = warm(window, Prefetch.parse(prefetchMode)); } @@ -94,9 +101,11 @@ public abstract class CoreVectors implements LongFunction { return null; } - /// Translates a reader-relative binding window to absolute record - /// coordinates: shifted by the profile window's start and clamped - /// to its end when the facet declares one. Returns the profile + /// Translates a binding window into dataset record coordinates for + /// the fetch. A profile-declared window re-bases the reader it + /// hands back, so a binding index is offset by that window's start + /// and clamped to its end; profiles in practice declare windows + /// starting at 0, where this is the identity. Returns the profile /// window itself when no binding window is given, [DSWindow#ALL] /// when neither applies, and `null` when the binding window falls /// entirely outside the profile window — nothing to warm. diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java index 98d7057f8..3c1f3dc18 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java @@ -24,41 +24,55 @@ import io.nosqlbench.vectordata.ElementType; import io.nosqlbench.vectordata.PrebufferProgress; import io.nosqlbench.vectordata.VectorReader; -/// Binding-layer clip of a [VectorReader] to a record window, with the -/// same semantics a profile-declared window gives a reader in -/// nb-vectordata (and in `vectordata-rs`): only the first interval -/// applies, and both bounds clamp to the underlying reader's count. -/// This adapter lives here rather than in nb-vectordata because the -/// vectordata crate exposes no caller-side reader windowing — windows -/// there come from facet configuration — and the binding functions' -/// window argument is a binding-layer convenience. +/// Restricts a [VectorReader] to a record range **without renumbering +/// it**: a window of `[50000..100000)` means indices 50000 through +/// 99999 are readable and everything else is refused. Indices stay +/// absolute, so the same ordinals mean the same records here as +/// everywhere else in this package. +/// +/// Used by the `windowedFacet` expression function, where a caller has +/// explicitly asked for a restricted reader. The binding functions do +/// **not** wrap their readers in this: a binding window only warms, and +/// restricting it would break resolution, since VirtData probes a newly +/// resolved function with a small sample index before any cycle runs. +/// +/// Only the first interval of the window applies, and both bounds clamp +/// to the underlying reader's count. public final class WindowedReader implements VectorReader { private final VectorReader inner; - private final long begin, count; + private final long begin, end; - /// Clips `reader` to the first interval of `window`, in the - /// reader's own index coordinates. A blank window returns the - /// reader unchanged. + /// Restricts `reader` to the first interval of `window`, in the + /// reader's own — absolute — index coordinates. A blank window + /// returns the reader unchanged. public static VectorReader clip(VectorReader reader, String window) { DSWindow parsed = DSWindow.parse(window == null ? "" : window); if (parsed.isEmpty()) return reader; DSWindow.Interval first = parsed.intervals().get(0); long total = reader.count(); - long begin = Math.min(first.minIncl(), total); - long count = Math.max(0, Math.min(first.maxExcl(), total) - begin); - return new WindowedReader<>(reader, begin, count); + return new WindowedReader<>(reader, Math.min(first.minIncl(), total), Math.min(first.maxExcl(), total)); } - private WindowedReader(VectorReader inner, long begin, long count) { - this.inner = inner; this.begin = begin; this.count = count; + private WindowedReader(VectorReader inner, long begin, long end) { + this.inner = inner; this.begin = begin; this.end = end; } - @Override public long count() { return count; } + /// The exclusive upper bound of the readable range — the highest + /// index this reader accepts, plus one. Indices below the window's + /// start are refused even though they are below this count, because + /// the window is a restriction rather than a re-basing. + @Override public long count() { return end; } + /// First index this reader accepts. + public long begin() { return begin; } @Override public int dimension() { return inner.dimension(); } - @Override public A get(long index) { check(index); return inner.get(begin + index); } - @Override public void get(long index, A target) { check(index); inner.get(begin + index, target); } - private void check(long index) { if (index < 0 || index >= count) throw new IndexOutOfBoundsException("windowed index " + index + " outside 0.." + (count - 1)); } + @Override public A get(long index) { check(index); return inner.get(index); } + @Override public void get(long index, A target) { check(index); inner.get(index, target); } + private void check(long index) { + if (index < begin || index >= end) + throw new IndexOutOfBoundsException("index " + index + " is outside the binding window [" + + begin + ".." + end + "); window indices are absolute record ordinals, so cycles must fall inside it"); + } @Override public void prebuffer(PrebufferProgress progress) { inner.prebuffer(progress); } @Override public boolean isComplete() { return inner.isComplete(); } @Override public CacheStats cacheStats() { return inner.cacheStats(); } diff --git a/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java b/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java index beaed80f3..46c2f56ca 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java +++ b/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java @@ -108,10 +108,31 @@ public class VectorDataBindingsTest { } @Test - void aWindowArgumentClipsTheReaderAndWarmsIt() { - BaseVectors clipped = new BaseVectors("example:demo", "1..3", "eager", settings); - assertArrayEquals(new float[] {1f, 2f}, clipped.apply(0), "windowed record 0 is underlying record 1"); - assertThrows(IndexOutOfBoundsException.class, () -> clipped.apply(2), "the window is two records long"); + void aWindowWarmsWithoutRenumbering() { + // The window says which records to warm; it does not re-base + // them. Index 1 is record 1, as it would be with no window. + BaseVectors warmed = new BaseVectors("example:demo", "1..3", "eager", settings); + assertArrayEquals(new float[] {1f, 2f}, warmed.apply(1), "index 1 is record 1, not the window's first record"); + assertArrayEquals(new float[] {2f, 3f}, warmed.apply(2)); + } + + @Test + void aWindowFarFromZeroReadsTheRecordsItsCyclesName() { + // The field failure: a window starting well above zero used to + // re-base to 0, so a run over those same cycles indexed past the + // end and threw. + BaseVectors slice = new BaseVectors("example:demo", "[7..10)", "none", settings); + assertArrayEquals(new float[] {7f, 8f}, slice.apply(7)); + assertArrayEquals(new float[] {9f, 10f}, slice.apply(9)); + } + + @Test + void readingOutsideTheWindowStillWorks() { + // Not an error, and load-bearing: VirtData probes a resolved + // function with a small sample index before any cycle runs, so a + // window starting above zero would otherwise fail to resolve. + BaseVectors slice = new BaseVectors("example:demo", "[7..10)", "none", settings); + assertArrayEquals(new float[] {1f, 2f}, slice.apply(1), "outside the warmed window, but readable"); } @Test @@ -119,16 +140,16 @@ public class VectorDataBindingsTest { BaseVectors vectors = new BaseVectors("example:demo", "1..3", "background", settings); assertNotNull(vectors.backgroundPrefetch(), "background mode hands back the running prefetch"); vectors.backgroundPrefetch().join(); - assertArrayEquals(new float[] {2f, 3f}, vectors.apply(1)); + assertArrayEquals(new float[] {1f, 2f}, vectors.apply(1)); } @Test void aBindingWindowComposesWithAProfileWindow() { - // The profile clips to records [2..8); the binding window is in - // the clipped reader's coordinates, so 1..3 reads records 3..5. + // The profile clips to records [2..8), which re-bases the reader + // it hands back; the binding window then restricts that reader's + // indices 1..2, and warming shifts them to dataset records 3..5. BaseVectors vectors = new BaseVectors("example:windowed", "1..3", "eager", settings); - assertArrayEquals(new float[] {3f, 4f}, vectors.apply(0)); - assertThrows(IndexOutOfBoundsException.class, () -> vectors.apply(2)); + assertArrayEquals(new float[] {3f, 4f}, vectors.apply(1)); // And warming shifts the same window to absolute records 3..5. DSWindow effective = CoreVectors.effectiveWindow(vectors.tdv, "base_vectors", "1..3"); assertEquals(new DSWindow.Interval(3, 5), effective.intervals().get(0));