From 0c59a581918da237a309369db11aa7855e8710cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:32:15 +0000 Subject: [PATCH 1/9] build(deps): bump softprops/action-gh-release from 3.0.2 to 3.0.3 Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/v3.0.2...v3.0.3) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: 3.0.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/preview.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 760230be7..8cafab30a 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -153,7 +153,7 @@ jobs: # https://github.com/softprops/action-gh-release - name: create github release - uses: softprops/action-gh-release@v3.0.2 + uses: softprops/action-gh-release@v3.0.3 if: startsWith(github.ref, 'refs/tags/') with: # body: ${{ steps.prepare_summary.outputs.release_summary }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ceb578e5b..7e1696683 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: # https://github.com/softprops/action-gh-release - name: create github release - uses: softprops/action-gh-release@v3.0.2 + uses: softprops/action-gh-release@v3.0.3 if: startsWith(github.ref, 'refs/tags/') with: # body: ${{ steps.prepare_summary.outputs.release_summary }} From 9085bf06ad9077ec1c8ff61202e6dcc3a78434fc Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 1 Sep 2026 18:51:09 +0000 Subject: [PATCH 2/9] feat(virtdata): prefetch progress, profile warm-up, and cycle ranges from exprs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A prefetch driven from a workload was silent and awkward to aim: - every expr prefetch now reports through the same meter the bindings use, so a download started from a workload announces its cost, emits progress while the bytes move, and confirms completion - prefetchProfile(spec) warms every facet the profile declares, each to the window it declares, in one call — the form that cannot accidentally pull a whole shared base file, since an empty window means the entire facet by design - prefetchCycles(spec, facet, start, end) takes the ordinal range directly, which is how an activity already describes its work The docs record the three things that bite: expressions evaluate at load time rather than when a tagged block runs, an empty window is a request for everything, and assignment sigils render their value where they are written. --- .../vectordata/vectordata_remote_access.md | 37 ++++++++++++-- .../baselinesv2/cql_vector_dataset.yaml | 11 +++-- .../exprs/lib/vectors/VectorDataExprs.java | 49 ++++++++++++++++++- .../lib/vectors/vectordata/PrefetchMeter.java | 17 ++++--- 4 files changed, 99 insertions(+), 15 deletions(-) diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index 9c478ae63..a422f769e 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -142,10 +142,15 @@ expr forms (`{{= expr}}`, `{{name = expr}}`, `{{@name}}`); anything else passes through untouched. The vectordata expr functions: ```yaml -# Precache the whole facet before the run (empty window = everything): -# {{= prefetch("example:default", "base_vectors", "").rangesFetched()}} +# One-call warm-up: every facet the profile declares, each to the +# window the profile declares for it. This is the form to reach for — +# it never fetches more than the profile describes. +# {{= prefetchProfile("example:default")}} -# Ordinal-range warm-up, priced first: +# A cycle range, in the coordinates an activity already speaks: +# {{= prefetchCycles("example:default", "base_vectors", 0, 100000).rangesFetched()}} + +# An explicit window, priced first: # {{= prefetchPlan("example:default", "base_vectors", "[0..100k)").bytesToFetch()}} # {{= prefetch("example:default", "base_vectors", "[0..100k)").rangesFetched()}} @@ -163,6 +168,32 @@ passes through untouched. The vectordata expr functions: Diagnose expr processing with `dryrun=exprs` on the activity to dump the expression-processed workload and context. +### Three things to know when prefetching from a workload + +**Expressions evaluate when the workload loads, not when a block runs.** +The whole document is expression-processed before ops are parsed, so a +prefetch written into an op body fires on every activity start — +including `run tags='block:drop'` — no matter what tags that op carries. +That is fine for an up-front warm-up, but it is not a way to tie +fetching to a phase. To warm only when a phase actually uses the data, +let the bindings do it: `BaseVectors('ds:profile','[0..1M)','eager')` +warms when that binding resolves, which happens only for blocks that use +it, and `'background'` lets the fetch overlap the run. + +**An empty window means the whole facet, not the profile's window.** +`prefetch(spec,"base_vectors","")` on a sized profile fetches the entire +shared base file — which may be terabytes — because an empty window is a +request for everything rather than a fallback. Use `prefetchProfile`, +which resolves each facet to its declared window, or pass the window +explicitly. + +**An assignment sigil renders its value.** `{{p = "ds:" + profile}}` +substitutes the assigned value into the document at that spot, so +writing assignments in an op body leaves stray text in the op. Put them +in `description:` (where they read as documentation) and reference them +with `{{@p}}` where the value belongs — the workload example below does +exactly this. + For a complete workload where every dataset-shaped parameter is derived this way, see `nb-adapters/adapter-cqld4/src/main/resources/activities/baselinesv2/cql_vector_dataset.yaml`: diff --git a/nb-adapters/adapter-cqld4/src/main/resources/activities/baselinesv2/cql_vector_dataset.yaml b/nb-adapters/adapter-cqld4/src/main/resources/activities/baselinesv2/cql_vector_dataset.yaml index 3293a139a..017df5db8 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/activities/baselinesv2/cql_vector_dataset.yaml +++ b/nb-adapters/adapter-cqld4/src/main/resources/activities/baselinesv2/cql_vector_dataset.yaml @@ -14,9 +14,14 @@ description: | maxk: {{maxk == neighborIndices(spec).dimension()}} similarity: {{similarity == dataset(spec).attributes().getOrDefault('distance_function','COSINE').toString().toLowerCase()}} base bytes to warm (profile window): {{warmbytes == prefetchPlan(spec, 'base_vectors', dataset(spec).facet('base_vectors').get().window() ?: '').bytesToFetch()}} - The rampup bindings warm the profile's base window eagerly before the - first cycle (with download progress on stderr); pass a window and - prefetch mode on the binding functions to shape that differently. + The bindings warm what they read, when they resolve — so a block that + does not use a facet never fetches it — with download progress on + stderr. Pass a window and prefetch mode to shape that: + BaseVectors('{{@spec}}','[0..1M)','background') warms a cycle range + while the run proceeds. To warm everything up front instead, run + prefetchProfile('{{@spec}}') from an expr, remembering that + expressions evaluate when the workload loads rather than when a block + runs. Operational knobs stay conventional: keyspace, table, rf, and cl are TEMPLATE parameters with the usual defaults. diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java index 65cfce2e8..46dea909d 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java @@ -25,6 +25,7 @@ import io.nosqlbench.nb.api.expr.annotations.ExprFunctionSpec; import io.nosqlbench.vectordata.Catalog; import io.nosqlbench.vectordata.CatalogSources; import io.nosqlbench.vectordata.DSWindow; +import io.nosqlbench.vectordata.FacetDescriptor; import io.nosqlbench.vectordata.PrefetchHandle; import io.nosqlbench.vectordata.PrefetchPlan; import io.nosqlbench.vectordata.PrefetchReport; @@ -32,6 +33,7 @@ import io.nosqlbench.vectordata.TestDataView; import io.nosqlbench.vectordata.VectorReader; import io.nosqlbench.vectordata.VvecReader; import io.nosqlbench.vectordata.WholeFacetFallback; +import io.nosqlbench.virtdata.lib.vectors.vectordata.PrefetchMeter; import io.nosqlbench.virtdata.lib.vectors.vectordata.WindowedReader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -160,7 +162,52 @@ public class VectorDataExprs implements ExprFunctionProvider { description = "Fetch a record window of a facet and return when it is resident. An empty window requests the whole facet; an unresolvable window is refused rather than fetching everything." ) public PrefetchReport prefetch(String datasetNameAndProfile, String facetName, String window) { - return dataset(datasetNameAndProfile).prefetch(facetName, DSWindow.parse(window), WholeFacetFallback.REFUSE); + return warm(dataset(datasetNameAndProfile), datasetNameAndProfile, facetName, DSWindow.parse(window)); + } + + @ExprExample(args = {"\"airports:demo\"", "\"base_vectors\"", "0", "1000"}, expectNotNull = true) + @ExprExample(args = {"\"airports:demo\"", "\"query_vectors\"", "0", "10"}, expectNotNull = true) + @ExprFunctionSpec( + name = "prefetchCycles", + synopsis = "prefetchCycles(\"dataset:profile\", \"facet_name\", start, end)", + description = "Fetch the records a cycle range will read — the half-open ordinal interval [start,end) — and return when they are resident. The natural form when the range comes from an activity's cycles." + ) + public PrefetchReport prefetchCycles(String datasetNameAndProfile, String facetName, Number start, Number end) { + DSWindow window = new DSWindow(java.util.List.of(new DSWindow.Interval(start.longValue(), end.longValue()))); + return warm(dataset(datasetNameAndProfile), datasetNameAndProfile, facetName, window); + } + + @ExprExample(args = {"\"airports:demo\""}, expectNotNull = true) + @ExprExample(args = {"\"airports:demo\""}, matches = ".+") + @ExprFunctionSpec( + name = "prefetchProfile", + synopsis = "prefetchProfile(\"dataset:profile\")", + description = "Fetch every facet the profile declares, each to the window the profile declares for it (or whole when it declares none), and return a per-facet summary. The one-call warm-up for a workload." + ) + public String prefetchProfile(String datasetNameAndProfile) { + TestDataView view = dataset(datasetNameAndProfile); + StringBuilder summary = new StringBuilder(); + for (String facetName : view.facets().keySet()) { + String declared = view.facet(facetName).map(FacetDescriptor::window).orElse(null); + DSWindow window = DSWindow.parse(declared == null ? "" : declared); + PrefetchPlan plan = view.prefetchPlan(facetName, window); + warm(view, datasetNameAndProfile, facetName, window); + if (!summary.isEmpty()) summary.append("; "); + summary.append(facetName).append(' ').append(declared == null ? "ALL" : declared).append(' ') + .append(plan.isResident() ? "resident" : PrefetchMeter.bytes(plan.bytesToFetch())); + } + return summary.toString(); + } + + /// Plans, announces, fetches with progress, and confirms — the one + /// path every expr-driven warm-up takes, so a download started from + /// a workload reports itself the same way one started from a + /// binding does. + private static PrefetchReport warm(TestDataView view, String spec, String facetName, DSWindow window) { + PrefetchMeter meter = new PrefetchMeter(spec + ":" + facetName, view.prefetchPlan(facetName, window)); + PrefetchReport report = view.prefetch(facetName, window, WholeFacetFallback.REFUSE, meter); + meter.complete(); + return report; } @ExprExample(args = {"\"airports:demo\"", "\"base_vectors\"", "\"[0..100)\""}, expectNotNull = true) diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/PrefetchMeter.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/PrefetchMeter.java index e27debce5..ee30f3292 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/PrefetchMeter.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/PrefetchMeter.java @@ -22,20 +22,21 @@ import io.nosqlbench.vectordata.PrebufferProgress; import io.nosqlbench.vectordata.PrefetchHandle; import io.nosqlbench.vectordata.PrefetchPlan; -/// Console meter for a binding-level prefetch: the plan's cost is -/// announced before any bytes move, progress is emitted during the -/// download (rate-limited), and completion is confirmed. Lines go to +/// Console meter for a prefetch driven from a binding or an expr: the +/// plan's cost is announced before any bytes move, progress is emitted +/// during the download (rate-limited), and completion is confirmed. +/// Lines go to /// stderr so they are visible regardless of logging configuration — /// this is a download meter, not a log event. Silent when the plan has /// nothing to fetch, so local and already-warm facets add no noise. -final class PrefetchMeter implements PrebufferProgress { +public final class PrefetchMeter implements PrebufferProgress { private static final long INTERVAL_NANOS = 2_000_000_000L; private final String label; private final long planned; private volatile long lastEmit; - PrefetchMeter(String label, PrefetchPlan plan) { + public PrefetchMeter(String label, PrefetchPlan plan) { this.label = label; this.planned = plan.bytesToFetch(); if (planned > 0) @@ -52,13 +53,13 @@ final class PrefetchMeter implements PrebufferProgress { } /// Confirms a completed blocking fetch. - void complete() { + public void complete() { if (planned > 0) System.err.printf("[vectordata] %s: fetch complete (%s)%n", label, bytes(planned)); } /// Follows a background prefetch on a daemon thread, emitting the /// handle's byte counter until the worker finishes or is cancelled. - void watch(PrefetchHandle handle) { + public void watch(PrefetchHandle handle) { if (planned == 0) return; Thread watcher = new Thread(() -> { while (!handle.isDone()) { @@ -72,7 +73,7 @@ final class PrefetchMeter implements PrebufferProgress { watcher.start(); } - static String bytes(long count) { + public static String bytes(long count) { if (count >= 1L << 30) return String.format("%.1f GiB", count / (double) (1L << 30)); if (count >= 1L << 20) return String.format("%.1f MiB", count / (double) (1L << 20)); if (count >= 1L << 10) return String.format("%.1f KiB", count / (double) (1L << 10)); From acc1034d1494468d1688d69885111a56e044b695 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 1 Sep 2026 21:36:06 +0000 Subject: [PATCH 3/9] docs(vectordata): runnable demo workload with a live download meter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vectordata_demo.yaml needs no database — the stdout driver exercises dataset access alone. It prints what it derived from the dataset, fetches a record range with the meter running, and reads records back, so a download can be watched from one command line. prefetch=false with mode=background moves the fetch off the load path and overlaps it with the cycles instead. Binding meter labels now carry the profile, matching the expr form, so both name the same thing in a run that uses both. --- devdocs/vectordata/vectordata_demo.yaml | 55 +++++++++++++++++++ .../vectordata/vectordata_remote_access.md | 35 ++++++++++++ .../lib/vectors/vectordata/CoreVectors.java | 2 +- .../lib/vectors/vectordata/VariableFacet.java | 2 +- 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 devdocs/vectordata/vectordata_demo.yaml diff --git a/devdocs/vectordata/vectordata_demo.yaml b/devdocs/vectordata/vectordata_demo.yaml new file mode 100644 index 000000000..f16de37f4 --- /dev/null +++ b/devdocs/vectordata/vectordata_demo.yaml @@ -0,0 +1,55 @@ +min_version: "5.25" + +description: | + Runnable demo of remote vectordata access with a live download meter. + No database required — this uses the stdout driver, so the only thing + it exercises is dataset access itself. + + nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=: + + Everything below is read from the dataset when this file loads: + + dataset: {{spec == paramOr('dataset','UNSET — pass dataset=:')}} + dimensions: {{dimensions == baseVectors(spec).dimension()}} + base records: {{basecount == baseVectors(spec).count()}} + query records: {{querycount == queryVectors(spec).count()}} + neighbors k: {{maxk == neighborIndices(spec).dimension()}} + similarity: {{similarity == dataset(spec).attributes().getOrDefault('distance_function','unspecified')}} + + The demo works within the first `records` ordinals of the base facet, + and something has to fetch them. That fetch is announced, metered + while it runs, and confirmed — all on stderr, which is where the + progress you are looking for shows up: + + records: {{records == paramOr('records','20000') as Long}} + warmed at load:{{= paramOr('prefetch','true') == 'true' ? ' ' + prefetchCycles(spec,'base_vectors',0,records).rangesFetched() + ' range(s)' : ' skipped (prefetch=false)'}} + + Two ways to watch it, both from the command line: + + # 1. warm at load, then read — the meter runs before any cycles + nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=: + + # 2. warm in the background while cycles run — turn off the + # load-time fetch so the binding is the one doing the work + nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=: \ + prefetch=false mode=background cycles=1000 + + Parameters: cycles (default 3), records (default 20000), prefetch + (true|false, default true), mode (none|eager|background, default + none). The meter only speaks when there are bytes to move, so a + second run against a warm cache is silent by design — point + VECTORDATA_HOME at an empty directory to watch it fetch again. + +scenarios: + default: >- + run driver=stdout format=readout cycles=TEMPLATE(cycles,3) threads=1 + +bindings: + cycle: Identity() + # The window keeps reads inside the demo's record range, and mode + # selects who warms it: 'none' leaves that to the load-time prefetch + # above, 'eager' blocks on it here, 'background' overlaps it with the + # run. + base_vector: BaseVectors('{{@spec}}','[0..{{@records}})','TEMPLATE(mode,none)'); Stringify() + query_vector: QueryVectors('{{@spec}}',false); Stringify() + neighbor_indices: NeighborIndices('{{@spec}}',false); Stringify() diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index a422f769e..9f1a20762 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -26,6 +26,41 @@ e.g. `NB='java -jar nb5.jar'`). `VECTORDATA_CATALOG` outranks any `VECTORDATA_HOME` under `/tmp/vdtest` so it never touches `~/.config/vectordata` or your real cache. +## Watch a download happen + +[`vectordata_demo.yaml`](vectordata_demo.yaml) is a runnable workload +that needs no database — it uses the stdout driver, so the only thing it +exercises is dataset access. It prints what it derived from the dataset, +fetches a record range with a live meter, and reads records back: + +```bash +export VECTORDATA_CATALOG='https://your.host/path/catalog.yaml' +export VECTORDATA_HOME=/tmp/vdtest # empty dir ⇒ nothing is cached yet +nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=mydataset:myprofile +``` + +The meter goes to stderr while the bytes move: + +``` +[vectordata] mydataset:myprofile:base_vectors: fetching 29.0 MiB in 1 range(s) +[vectordata] mydataset:myprofile:base_vectors: 12.0 MiB / 30.0 MiB +[vectordata] mydataset:myprofile:base_vectors: fetch complete (29.0 MiB) +``` + +To watch the fetch overlap the run instead of preceding it, turn off the +load-time prefetch and let the binding warm in the background: + +```bash +nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=mydataset:myprofile \ + prefetch=false mode=background cycles=1000 +``` + +Parameters: `cycles`, `records` (how many ordinals the demo spans), +`prefetch` (`true|false`), and `mode` (`none|eager|background`). The +meter is silent when there is nothing to fetch, so a second run against +a warm cache prints nothing — point `VECTORDATA_HOME` at an empty +directory to see it fetch again. + Beyond the script's checks: ```bash diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java index d534b1114..53ebd824b 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java @@ -83,7 +83,7 @@ public abstract class CoreVectors implements LongFunction { // The plan is announced before any bytes move and progress is // emitted during the download; the meter is silent when there // is nothing to fetch. - PrefetchMeter meter = new PrefetchMeter(tdv.dataset() + ":" + facetName, tdv.prefetchPlan(facetName, effective)); + PrefetchMeter meter = new PrefetchMeter(tdv.dataset() + ":" + tdv.profile() + ":" + facetName, tdv.prefetchPlan(facetName, effective)); if (mode == Prefetch.BACKGROUND) { PrefetchHandle handle = tdv.prefetchInBackground(facetName, effective, WholeFacetFallback.REFUSE); meter.watch(handle); diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VariableFacet.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VariableFacet.java index 99dbd9258..5e85fa4f8 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VariableFacet.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VariableFacet.java @@ -78,7 +78,7 @@ public class VariableFacet implements LongFunction { DSWindow parsed = DSWindow.parse(window == null ? "" : window); CoreVectors.Prefetch mode = CoreVectors.Prefetch.parse(prefetchMode); PrefetchMeter meter = mode == CoreVectors.Prefetch.NONE ? null - : new PrefetchMeter(tdv.dataset() + ":" + this.facetName, tdv.prefetchPlan(this.facetName, parsed)); + : new PrefetchMeter(tdv.dataset() + ":" + tdv.profile() + ":" + this.facetName, tdv.prefetchPlan(this.facetName, parsed)); backgroundPrefetch = switch (mode) { case NONE -> null; case BACKGROUND -> { From 0edf2abd5ab6e8170c72f0edd8848cd862951720 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 1 Sep 2026 22:12:13 +0000 Subject: [PATCH 4/9] version bump --- mvn-defaults/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 59937c56c..dd1157552 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -25,7 +25,7 @@ pom - 5.25.14-SNAPSHOT + 5.25.15-SNAPSHOT INFO From 0b9e16ee2097bdbe42f6645c0f0c419f1d93ea5a Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 1 Sep 2026 23:08:10 +0000 Subject: [PATCH 5/9] feat(stdout): bundle the vectordata demo as a runnable workload The demo only worked from a checkout. It now ships in the stdout adapter's activities, so 'nb5 run vectordata_demo dataset=:' works from the distribution alone, with --cat and --copy for reading or editing it. Every catalog-touching expression is guarded on the dataset parameter being given. Bundled workloads are parsed through the expression pipeline by --list-workloads and --list-scenarios, so a workload that opened a catalog unconditionally would fail listing for every user without one configured; with no dataset named it now renders usage text and touches nothing. --- devdocs/vectordata/vectordata_demo.yaml | 55 ---------------- .../vectordata/vectordata_remote_access.md | 15 +++-- .../activities/examples/vectordata_demo.yaml | 64 +++++++++++++++++++ 3 files changed, 73 insertions(+), 61 deletions(-) delete mode 100644 devdocs/vectordata/vectordata_demo.yaml create mode 100644 nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml diff --git a/devdocs/vectordata/vectordata_demo.yaml b/devdocs/vectordata/vectordata_demo.yaml deleted file mode 100644 index f16de37f4..000000000 --- a/devdocs/vectordata/vectordata_demo.yaml +++ /dev/null @@ -1,55 +0,0 @@ -min_version: "5.25" - -description: | - Runnable demo of remote vectordata access with a live download meter. - No database required — this uses the stdout driver, so the only thing - it exercises is dataset access itself. - - nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=: - - Everything below is read from the dataset when this file loads: - - dataset: {{spec == paramOr('dataset','UNSET — pass dataset=:')}} - dimensions: {{dimensions == baseVectors(spec).dimension()}} - base records: {{basecount == baseVectors(spec).count()}} - query records: {{querycount == queryVectors(spec).count()}} - neighbors k: {{maxk == neighborIndices(spec).dimension()}} - similarity: {{similarity == dataset(spec).attributes().getOrDefault('distance_function','unspecified')}} - - The demo works within the first `records` ordinals of the base facet, - and something has to fetch them. That fetch is announced, metered - while it runs, and confirmed — all on stderr, which is where the - progress you are looking for shows up: - - records: {{records == paramOr('records','20000') as Long}} - warmed at load:{{= paramOr('prefetch','true') == 'true' ? ' ' + prefetchCycles(spec,'base_vectors',0,records).rangesFetched() + ' range(s)' : ' skipped (prefetch=false)'}} - - Two ways to watch it, both from the command line: - - # 1. warm at load, then read — the meter runs before any cycles - nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=: - - # 2. warm in the background while cycles run — turn off the - # load-time fetch so the binding is the one doing the work - nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=: \ - prefetch=false mode=background cycles=1000 - - Parameters: cycles (default 3), records (default 20000), prefetch - (true|false, default true), mode (none|eager|background, default - none). The meter only speaks when there are bytes to move, so a - second run against a warm cache is silent by design — point - VECTORDATA_HOME at an empty directory to watch it fetch again. - -scenarios: - default: >- - run driver=stdout format=readout cycles=TEMPLATE(cycles,3) threads=1 - -bindings: - cycle: Identity() - # The window keeps reads inside the demo's record range, and mode - # selects who warms it: 'none' leaves that to the load-time prefetch - # above, 'eager' blocks on it here, 'background' overlaps it with the - # run. - base_vector: BaseVectors('{{@spec}}','[0..{{@records}})','TEMPLATE(mode,none)'); Stringify() - query_vector: QueryVectors('{{@spec}}',false); Stringify() - neighbor_indices: NeighborIndices('{{@spec}}',false); Stringify() diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index 9f1a20762..8953417e5 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -28,17 +28,20 @@ e.g. `NB='java -jar nb5.jar'`). `VECTORDATA_CATALOG` outranks any ## Watch a download happen -[`vectordata_demo.yaml`](vectordata_demo.yaml) is a runnable workload -that needs no database — it uses the stdout driver, so the only thing it -exercises is dataset access. It prints what it derived from the dataset, -fetches a record range with a live meter, and reads records back: +`vectordata_demo` is a workload bundled in nb5 — no checkout, no +database. It uses the stdout driver, so the only thing it exercises is +dataset access: it prints what it derived from the dataset, fetches a +record range with a live meter, and reads records back. ```bash export VECTORDATA_CATALOG='https://your.host/path/catalog.yaml' export VECTORDATA_HOME=/tmp/vdtest # empty dir ⇒ nothing is cached yet -nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=mydataset:myprofile +nb5 run vectordata_demo dataset=mydataset:myprofile ``` +`nb5 --cat vectordata_demo` prints it and `nb5 --copy vectordata_demo` +drops an editable copy in the working directory. + The meter goes to stderr while the bytes move: ``` @@ -51,7 +54,7 @@ To watch the fetch overlap the run instead of preceding it, turn off the load-time prefetch and let the binding warm in the background: ```bash -nb5 run devdocs/vectordata/vectordata_demo.yaml dataset=mydataset:myprofile \ +nb5 run vectordata_demo dataset=mydataset:myprofile \ prefetch=false mode=background cycles=1000 ``` diff --git a/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml b/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml new file mode 100644 index 000000000..62c88e2a9 --- /dev/null +++ b/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml @@ -0,0 +1,64 @@ +min_version: "5.25" + +description: | + Remote vector dataset access with a live download meter. No database + required — this uses the stdout driver, so the only thing exercised is + dataset access itself. + + nb5 run vectordata_demo dataset=: + + Point it at a catalog first, either of these: + + export VECTORDATA_CATALOG='https://your.host/path/catalog.yaml' + nb5 run vectordata_demo dataset=: -Dvectordata.catalog= + + Everything the workload needs is read from the dataset itself when the + file loads — dimensions, record counts, k, and the similarity function + come from the data rather than from parameters you have to restate. + + dataset: {{spec == paramOr('dataset','MISSING-dataset-parameter')}} + records: {{records == paramOr('records','20000') as Long}} + {{= spec == 'MISSING-dataset-parameter' ? ' (pass dataset=: to derive the rest from a dataset)' : + ' dimensions: ' + baseVectors(spec).dimension() + '\n' + + ' base records: ' + baseVectors(spec).count() + '\n' + + ' query records: ' + queryVectors(spec).count() + '\n' + + ' neighbors k: ' + neighborIndices(spec).dimension() + '\n' + + ' similarity: ' + dataset(spec).attributes().getOrDefault('distance_function','unspecified')}} + + The demo works within the first `records` ordinals of the base facet, + and something has to fetch them. That fetch is announced, metered + while it runs, and confirmed — all on stderr, which is where the + progress shows up: + + {{= spec == 'MISSING-dataset-parameter' || paramOr('prefetch','true') != 'true' ? ' warm at load: skipped' : + ' warm at load: ' + prefetchCycles(spec,'base_vectors',0,records).rangesFetched() + ' range(s) fetched'}} + + Two ways to watch it: + + # 1. warm at load, then read — the meter runs before any cycles + nb5 run vectordata_demo dataset=: + + # 2. warm in the background while cycles run — turn off the + # load-time fetch so the binding does the work instead + nb5 run vectordata_demo dataset=: \ + prefetch=false mode=background cycles=1000 + + Parameters: dataset (required), cycles (default 3), records (default + 20000), prefetch (true|false, default true), mode (none|eager| + background, default none). The meter only speaks when there are bytes + to move, so a second run against a warm cache is silent by design — + point VECTORDATA_HOME at an empty directory to watch it fetch again. + +scenarios: + default: >- + run driver=stdout format=readout cycles=TEMPLATE(cycles,3) threads=1 + +bindings: + cycle: Identity() + # The window keeps reads inside the demo's record range, and mode + # selects who warms it: 'none' leaves that to the load-time prefetch + # above, 'eager' blocks on it here, 'background' overlaps it with the + # run. + base_vector: BaseVectors('{{@spec}}','[0..{{@records}})','TEMPLATE(mode,none)'); Stringify() + query_vector: QueryVectors('{{@spec}}',false); Stringify() + neighbor_indices: NeighborIndices('{{@spec}}',false); Stringify() From 66fc07c15febe43a14311bf86178722bad8074a3 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 2 Sep 2026 01:04:19 +0000 Subject: [PATCH 6/9] docs(stdout): correct the demo workload invocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The documented 'nb5 run vectordata_demo' does not work: run is itself a command, so the workload name is parsed as a second one and resolves to no service. The scenario form drops the run — 'nb5 vectordata_demo dataset=:' — and the long form names the full bundled path, 'nb5 run workload=activities/examples/vectordata_demo.yaml'. Both are now shown in the workload and the guide. Verified against the 5.25.15-release binary: the workload is bundled and listed, both corrected forms load it, and a real dataset run meters the download and reads base, query, and neighbor records. --- devdocs/vectordata/vectordata_remote_access.md | 12 ++++++++++-- .../activities/examples/vectordata_demo.yaml | 13 +++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index 8953417e5..731276c8c 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -36,7 +36,15 @@ record range with a live meter, and reads records back. ```bash export VECTORDATA_CATALOG='https://your.host/path/catalog.yaml' export VECTORDATA_HOME=/tmp/vdtest # empty dir ⇒ nothing is cached yet -nb5 run vectordata_demo dataset=mydataset:myprofile +nb5 vectordata_demo dataset=mydataset:myprofile +``` + +That is the scenario form — the workload name stands alone, with no +`run` in front of it (`run` is itself a command, so `nb5 run +vectordata_demo` fails). The equivalent long form names the full path: + +```bash +nb5 run workload=activities/examples/vectordata_demo.yaml dataset=mydataset:myprofile ``` `nb5 --cat vectordata_demo` prints it and `nb5 --copy vectordata_demo` @@ -54,7 +62,7 @@ To watch the fetch overlap the run instead of preceding it, turn off the load-time prefetch and let the binding warm in the background: ```bash -nb5 run vectordata_demo dataset=mydataset:myprofile \ +nb5 vectordata_demo dataset=mydataset:myprofile \ prefetch=false mode=background cycles=1000 ``` diff --git a/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml b/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml index 62c88e2a9..1c603aa83 100644 --- a/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml +++ b/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml @@ -5,12 +5,17 @@ description: | required — this uses the stdout driver, so the only thing exercised is dataset access itself. - nb5 run vectordata_demo dataset=: + nb5 vectordata_demo dataset=: + + That is the scenario form — the workload name stands alone, without a + `run` in front of it. The equivalent long form names the full path: + + nb5 run workload=activities/examples/vectordata_demo.yaml dataset=: Point it at a catalog first, either of these: export VECTORDATA_CATALOG='https://your.host/path/catalog.yaml' - nb5 run vectordata_demo dataset=: -Dvectordata.catalog= + nb5 vectordata_demo dataset=: -Dvectordata.catalog= Everything the workload needs is read from the dataset itself when the file loads — dimensions, record counts, k, and the similarity function @@ -36,11 +41,11 @@ description: | Two ways to watch it: # 1. warm at load, then read — the meter runs before any cycles - nb5 run vectordata_demo dataset=: + nb5 vectordata_demo dataset=: # 2. warm in the background while cycles run — turn off the # load-time fetch so the binding does the work instead - nb5 run vectordata_demo dataset=: \ + nb5 vectordata_demo dataset=: \ prefetch=false mode=background cycles=1000 Parameters: dataset (required), cycles (default 3), records (default From 52dc31bfcea6d16e90b088bc73ae27e15685e684 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 2 Sep 2026 15:23:17 +0000 Subject: [PATCH 7/9] feat(virtdata): meter the background prefetch and reach it directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asking for a prefetch with progress meant asking for prefetch=false and then a binding mode — two knobs that read as contradicting each other, because the background expr function was the one prefetch form that never got a meter, so the only way to see a background download was to route it through a binding. prefetchBackground now announces its plan and watches the handle like every other form, and prefetchCyclesBackground takes the ordinal range directly. With the API reachable at the surface, the demo asks for what it wants in one word: prefetch=load blocks on prefetchCycles before the run, prefetch=background hands back a metered handle that fetches while the cycles run, prefetch=none leaves reads to demand-page. The binding mode knob is gone. --- .../vectordata/vectordata_remote_access.md | 25 ++++++--- .../activities/examples/vectordata_demo.yaml | 56 +++++++++++-------- .../exprs/lib/vectors/VectorDataExprs.java | 23 +++++++- 3 files changed, 70 insertions(+), 34 deletions(-) diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index 731276c8c..4854da000 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -58,19 +58,23 @@ The meter goes to stderr while the bytes move: [vectordata] mydataset:myprofile:base_vectors: fetch complete (29.0 MiB) ``` -To watch the fetch overlap the run instead of preceding it, turn off the -load-time prefetch and let the binding warm in the background: +To watch the fetch overlap the run instead of preceding it, ask for the +background form — give it a range big enough and enough cycles to +outlast the download: ```bash nb5 vectordata_demo dataset=mydataset:myprofile \ - prefetch=false mode=background cycles=1000 + prefetch=background records=100000 cycles=200 ``` -Parameters: `cycles`, `records` (how many ordinals the demo spans), -`prefetch` (`true|false`), and `mode` (`none|eager|background`). The -meter is silent when there is nothing to fetch, so a second run against -a warm cache prints nothing — point `VECTORDATA_HOME` at an empty -directory to see it fetch again. +`prefetch` selects which prefetch call does the work: `load` (the +default) blocks on `prefetchCycles` before cycle 0, `background` hands +back a handle from `prefetchCyclesBackground` and meters it while the +run proceeds, and `none` leaves reads to demand-page. `records` sets how +many ordinals the demo spans, `cycles` how long it runs. The meter is +silent when there is nothing to fetch, so a second run against a warm +cache prints nothing — point `VECTORDATA_HOME` at an empty directory to +see it fetch again. Beyond the script's checks: @@ -200,7 +204,10 @@ passes through untouched. The vectordata expr functions: # {{= prefetchPlan("example:default", "base_vectors", "[0..100k)").bytesToFetch()}} # {{= prefetch("example:default", "base_vectors", "[0..100k)").rangesFetched()}} -# Background warm-up captured in an expr variable, joined later: +# The same, in the background: the handle comes back immediately and +# the fetch overlaps whatever runs next. Both background forms meter +# from the handle, so the download reports itself either way. +# {{= prefetchCyclesBackground("example:default", "base_vectors", 0, 1000000).plan().requests()}} # {{warmup = prefetchBackground("example:default", "base_vectors", "[0..1M)")}} # {{= warmup.join().rangesFetched()}} diff --git a/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml b/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml index 1c603aa83..7711c668b 100644 --- a/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml +++ b/nb-adapters/adapter-stdout/src/main/resources/activities/examples/vectordata_demo.yaml @@ -7,11 +7,6 @@ description: | nb5 vectordata_demo dataset=: - That is the scenario form — the workload name stands alone, without a - `run` in front of it. The equivalent long form names the full path: - - nb5 run workload=activities/examples/vectordata_demo.yaml dataset=: - Point it at a catalog first, either of these: export VECTORDATA_CATALOG='https://your.host/path/catalog.yaml' @@ -23,6 +18,7 @@ description: | dataset: {{spec == paramOr('dataset','MISSING-dataset-parameter')}} records: {{records == paramOr('records','20000') as Long}} + prefetch: {{prefetchMode == paramOr('prefetch','load')}} {{= spec == 'MISSING-dataset-parameter' ? ' (pass dataset=: to derive the rest from a dataset)' : ' dimensions: ' + baseVectors(spec).dimension() + '\n' + ' base records: ' + baseVectors(spec).count() + '\n' + @@ -30,29 +26,42 @@ description: | ' neighbors k: ' + neighborIndices(spec).dimension() + '\n' + ' similarity: ' + dataset(spec).attributes().getOrDefault('distance_function','unspecified')}} - The demo works within the first `records` ordinals of the base facet, - and something has to fetch them. That fetch is announced, metered - while it runs, and confirmed — all on stderr, which is where the - progress shows up: + The demo reads the first `records` ordinals of the base facet, and + `prefetch` chooses which prefetch call fetches them. Both fetching + forms report the same way — the plan is announced, progress is metered + while the bytes move, and completion is confirmed, all on stderr: - {{= spec == 'MISSING-dataset-parameter' || paramOr('prefetch','true') != 'true' ? ' warm at load: skipped' : - ' warm at load: ' + prefetchCycles(spec,'base_vectors',0,records).rangesFetched() + ' range(s) fetched'}} + prefetch=load prefetchCycles(...) — fetches before the run + starts, so cycle 0 reads warm data. Blocks + until resident. This is the default. + prefetch=background prefetchCyclesBackground(...) — returns a + handle immediately and fetches while the cycles + run, so reads and the download overlap. + Progress is polled from the handle. Give it + enough cycles to outlast the download, or the + run ends first and the fetch stops partway. + prefetch=none no warm-up; reads demand-page whatever they + touch. Nothing to meter, so it stays quiet. + + {{= spec == 'MISSING-dataset-parameter' ? ' warm-up: skipped (no dataset given)' : + prefetchMode == 'load' ? ' warm-up: fetched ' + prefetchCycles(spec,'base_vectors',0,records).rangesFetched() + ' range(s) before the run' : + prefetchMode == 'background' ? ' warm-up: started in the background, ' + prefetchCyclesBackground(spec,'base_vectors',0,records).plan().requests() + ' range(s) planned' : + ' warm-up: none, reads demand-page'}} Two ways to watch it: - # 1. warm at load, then read — the meter runs before any cycles + # fetch up front, then read — the meter runs before any cycles nb5 vectordata_demo dataset=: - # 2. warm in the background while cycles run — turn off the - # load-time fetch so the binding does the work instead + # fetch during the run — a large range and enough cycles to see it nb5 vectordata_demo dataset=: \ - prefetch=false mode=background cycles=1000 + prefetch=background records=100000 cycles=200 Parameters: dataset (required), cycles (default 3), records (default - 20000), prefetch (true|false, default true), mode (none|eager| - background, default none). The meter only speaks when there are bytes - to move, so a second run against a warm cache is silent by design — - point VECTORDATA_HOME at an empty directory to watch it fetch again. + 20000), prefetch (load|background|none, default load). The meter only + speaks when there are bytes to move, so a second run against a warm + cache is silent by design — point VECTORDATA_HOME at an empty + directory to watch it fetch again. scenarios: default: >- @@ -60,10 +69,9 @@ scenarios: bindings: cycle: Identity() - # The window keeps reads inside the demo's record range, and mode - # selects who warms it: 'none' leaves that to the load-time prefetch - # above, 'eager' blocks on it here, 'background' overlaps it with the - # run. - base_vector: BaseVectors('{{@spec}}','[0..{{@records}})','TEMPLATE(mode,none)'); Stringify() + # Reads are demand-paged here on purpose: the prefetch above is what + # warms the data, so the meter has one owner and this window only + # clips what the bindings expose. + base_vector: BaseVectors('{{@spec}}','[0..{{@records}})','none'); Stringify() query_vector: QueryVectors('{{@spec}}',false); Stringify() neighbor_indices: NeighborIndices('{{@spec}}',false); Stringify() diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java index 46dea909d..b8b16a94f 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java @@ -218,7 +218,28 @@ public class VectorDataExprs implements ExprFunctionProvider { description = "Start fetching a record window of a facet on another thread and return the handle; join() waits, cancel() stops between ranges." ) public PrefetchHandle prefetchBackground(String datasetNameAndProfile, String facetName, String window) { - return dataset(datasetNameAndProfile).prefetchInBackground(facetName, DSWindow.parse(window), WholeFacetFallback.REFUSE); + TestDataView view = dataset(datasetNameAndProfile); + DSWindow parsed = DSWindow.parse(window); + // Metered like every other prefetch form: the plan is announced + // before the handle is returned, and a watcher reports the + // handle's byte counter while the fetch runs alongside whatever + // the caller does next. + PrefetchMeter meter = new PrefetchMeter(datasetNameAndProfile + ":" + facetName, view.prefetchPlan(facetName, parsed)); + PrefetchHandle handle = view.prefetchInBackground(facetName, parsed, WholeFacetFallback.REFUSE); + meter.watch(handle); + return handle; + } + + @ExprExample(args = {"\"airports:demo\"", "\"base_vectors\"", "0", "1000"}, expectNotNull = true) + @ExprExample(args = {"\"airports:demo\"", "\"query_vectors\"", "0", "10"}, expectNotNull = true) + @ExprFunctionSpec( + name = "prefetchCyclesBackground", + synopsis = "prefetchCyclesBackground(\"dataset:profile\", \"facet_name\", start, end)", + description = "Start fetching the records a cycle range will read, on another thread, and return the handle. The background counterpart of prefetchCycles, metered the same way." + ) + public PrefetchHandle prefetchCyclesBackground(String datasetNameAndProfile, String facetName, Number start, Number end) { + return prefetchBackground(datasetNameAndProfile, facetName, + "[" + start.longValue() + ".." + end.longValue() + ")"); } } From 59d604aed671eaa772221ce1c8ce3c1128308db4 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 2 Sep 2026 19:36:03 +0000 Subject: [PATCH 8/9] fix(virtdata): binding windows warm records without renumbering them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A binding window re-based the reader to zero, so BaseVectors(spec, '[50000..100000)') exposed indices 0..49999. A workload running the cycles that window names then indexed past the end and failed, and it only ever worked from cycle 0, where the re-basing is the identity. It also disagreed with the two surfaces beside it: VariableFacet and prefetchCycles both speak absolute ordinals. The window now selects which records to warm and nothing else. Indices are absolute everywhere, so cycles address the records they name, and the fetch still covers only the window — a [50000..100000) slice of a 100k profile warms 74 MiB rather than the full 146 MiB. Reads outside the window demand-page rather than failing, which is load-bearing rather than lenient: VirtData probes a newly resolved function with a small sample index before any cycle runs, so refusing out-of-window reads leaves a window above zero unable to resolve at all. A test pins that. WindowedReader keeps its restricting behaviour for the windowedFacet expression function, where a caller asks for a restricted reader directly and no resolver probe is involved. --- .../vectordata/vectordata_remote_access.md | 15 +++-- .../exprs/lib/vectors/VectorDataExprs.java | 2 +- .../lib/vectors/vectordata/CoreVectors.java | 33 +++++++---- .../vectors/vectordata/WindowedReader.java | 56 ++++++++++++------- .../vectordata/VectorDataBindingsTest.java | 39 ++++++++++--- 5 files changed, 98 insertions(+), 47 deletions(-) diff --git a/devdocs/vectordata/vectordata_remote_access.md b/devdocs/vectordata/vectordata_remote_access.md index 4854da000..75056e352 100644 --- a/devdocs/vectordata/vectordata_remote_access.md +++ b/devdocs/vectordata/vectordata_remote_access.md @@ -296,10 +296,17 @@ java -jar nb5.jar run driver=stdout cycles=10 \ ``` Prefetch modes on every mapper: `eager` (default; aliases `prebuffer`, -`true`), `background`, `none` (aliases `demand`, `false`). A binding -window is in the reader's own coordinates — after any profile window — -and is shifted to absolute records for the fetch, so the warmed bytes -are exactly the bytes the clipped reader exposes. +`true`), `background`, `none` (aliases `demand`, `false`). + +A binding window says **which records to warm**, and nothing else — +indices stay absolute record ordinals. +`BaseVectors('ds:profile','[50000..100000)')` warms records 50000 +through 99999, and a run of `cycles=50000..100000` addresses exactly +those records. Reading outside the window is not an error; it just +demand-pages, as it would with no window. Ordinals mean the same thing +here, in `VariableFacet`, and in `prefetchCycles`. The window is +translated into dataset coordinates for the fetch, so the warmed bytes +are the bytes the reader will expose. ## Verifying that warming happened diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java index b8b16a94f..df65874b6 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java @@ -126,7 +126,7 @@ public class VectorDataExprs implements ExprFunctionProvider { @ExprFunctionSpec( name = "windowedFacet", synopsis = "windowedFacet(\"dataset:profile\", \"facet_name\", \"window\")", - description = "Return a reader for a named facet, clipped to a record window in the reader's own coordinates." + description = "Return a reader for a named facet, restricted to a record window. Indices stay absolute — a window of [50000..100000) accepts 50000..99999 — so cycle values address the records they name." ) public VectorReader windowedFacet(String datasetNameAndProfile, String facetName, String window) { return WindowedReader.clip(dataset(datasetNameAndProfile).openFacet(facetName), window); diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java index 53ebd824b..090d9ef99 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java @@ -35,9 +35,8 @@ import java.util.Locale; import java.util.function.LongFunction; /// Base plumbing for vectordata-backed binding functions: opens the -/// named dataset profile from the default catalogs, clips the reader to -/// an optional record window, and warms data according to a prefetch -/// mode: +/// named dataset profile from the default catalogs and warms an +/// optional record window according to a prefetch mode: /// /// - `eager` (aliases `prebuffer`, `true`) — fetch the effective window, /// or the whole facet when no window applies, before returning; @@ -45,11 +44,18 @@ import java.util.function.LongFunction; /// immediately, letting reads overlap the download; /// - `none` (aliases `demand`, `false`) — demand-paged access only. /// -/// A binding window is expressed in the reader's own index coordinates, -/// after any profile-declared window. For warming, it is translated to -/// absolute record coordinates by shifting against the profile window, -/// so the bytes fetched are exactly the bytes the clipped reader -/// exposes. +/// A binding window selects **which records to warm**, and nothing +/// else: indices stay absolute record ordinals, so a workload running +/// `cycles=50000..100000` with a window of `[50000..100000)` addresses +/// the records those cycles name. Reads outside the window are not an +/// error — they simply demand-page, the way they would with no window +/// at all — which also keeps the binding resolvable, since VirtData +/// probes a newly resolved function with a small sample index before +/// any cycle runs. Ordinals mean the same thing here, in +/// [VariableFacet], and in the `prefetchCycles` expression function. +/// The window is translated into dataset coordinates against any +/// profile-declared window, so the bytes fetched are the bytes the +/// reader will expose. public abstract class CoreVectors implements LongFunction { protected final TestDataView tdv; @@ -66,7 +72,8 @@ public abstract class CoreVectors implements LongFunction { Catalog catalog = Catalog.of(CatalogSources.defaults(), settings); tdv = catalog.openProfile(datasetAndProfile); this.facetName = FacetNames.canonical(facetName); - dataset = WindowedReader.clip(getRandomAccessData(), window); + // Deliberately unwrapped: the window warms, it does not clip. + dataset = getRandomAccessData(); backgroundPrefetch = warm(window, Prefetch.parse(prefetchMode)); } @@ -94,9 +101,11 @@ public abstract class CoreVectors implements LongFunction { return null; } - /// Translates a reader-relative binding window to absolute record - /// coordinates: shifted by the profile window's start and clamped - /// to its end when the facet declares one. Returns the profile + /// Translates a binding window into dataset record coordinates for + /// the fetch. A profile-declared window re-bases the reader it + /// hands back, so a binding index is offset by that window's start + /// and clamped to its end; profiles in practice declare windows + /// starting at 0, where this is the identity. Returns the profile /// window itself when no binding window is given, [DSWindow#ALL] /// when neither applies, and `null` when the binding window falls /// entirely outside the profile window — nothing to warm. diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java index 98d7057f8..3c1f3dc18 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/WindowedReader.java @@ -24,41 +24,55 @@ import io.nosqlbench.vectordata.ElementType; import io.nosqlbench.vectordata.PrebufferProgress; import io.nosqlbench.vectordata.VectorReader; -/// Binding-layer clip of a [VectorReader] to a record window, with the -/// same semantics a profile-declared window gives a reader in -/// nb-vectordata (and in `vectordata-rs`): only the first interval -/// applies, and both bounds clamp to the underlying reader's count. -/// This adapter lives here rather than in nb-vectordata because the -/// vectordata crate exposes no caller-side reader windowing — windows -/// there come from facet configuration — and the binding functions' -/// window argument is a binding-layer convenience. +/// Restricts a [VectorReader] to a record range **without renumbering +/// it**: a window of `[50000..100000)` means indices 50000 through +/// 99999 are readable and everything else is refused. Indices stay +/// absolute, so the same ordinals mean the same records here as +/// everywhere else in this package. +/// +/// Used by the `windowedFacet` expression function, where a caller has +/// explicitly asked for a restricted reader. The binding functions do +/// **not** wrap their readers in this: a binding window only warms, and +/// restricting it would break resolution, since VirtData probes a newly +/// resolved function with a small sample index before any cycle runs. +/// +/// Only the first interval of the window applies, and both bounds clamp +/// to the underlying reader's count. public final class WindowedReader implements VectorReader { private final VectorReader inner; - private final long begin, count; + private final long begin, end; - /// Clips `reader` to the first interval of `window`, in the - /// reader's own index coordinates. A blank window returns the - /// reader unchanged. + /// Restricts `reader` to the first interval of `window`, in the + /// reader's own — absolute — index coordinates. A blank window + /// returns the reader unchanged. public static VectorReader clip(VectorReader reader, String window) { DSWindow parsed = DSWindow.parse(window == null ? "" : window); if (parsed.isEmpty()) return reader; DSWindow.Interval first = parsed.intervals().get(0); long total = reader.count(); - long begin = Math.min(first.minIncl(), total); - long count = Math.max(0, Math.min(first.maxExcl(), total) - begin); - return new WindowedReader<>(reader, begin, count); + return new WindowedReader<>(reader, Math.min(first.minIncl(), total), Math.min(first.maxExcl(), total)); } - private WindowedReader(VectorReader inner, long begin, long count) { - this.inner = inner; this.begin = begin; this.count = count; + private WindowedReader(VectorReader inner, long begin, long end) { + this.inner = inner; this.begin = begin; this.end = end; } - @Override public long count() { return count; } + /// The exclusive upper bound of the readable range — the highest + /// index this reader accepts, plus one. Indices below the window's + /// start are refused even though they are below this count, because + /// the window is a restriction rather than a re-basing. + @Override public long count() { return end; } + /// First index this reader accepts. + public long begin() { return begin; } @Override public int dimension() { return inner.dimension(); } - @Override public A get(long index) { check(index); return inner.get(begin + index); } - @Override public void get(long index, A target) { check(index); inner.get(begin + index, target); } - private void check(long index) { if (index < 0 || index >= count) throw new IndexOutOfBoundsException("windowed index " + index + " outside 0.." + (count - 1)); } + @Override public A get(long index) { check(index); return inner.get(index); } + @Override public void get(long index, A target) { check(index); inner.get(index, target); } + private void check(long index) { + if (index < begin || index >= end) + throw new IndexOutOfBoundsException("index " + index + " is outside the binding window [" + + begin + ".." + end + "); window indices are absolute record ordinals, so cycles must fall inside it"); + } @Override public void prebuffer(PrebufferProgress progress) { inner.prebuffer(progress); } @Override public boolean isComplete() { return inner.isComplete(); } @Override public CacheStats cacheStats() { return inner.cacheStats(); } diff --git a/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java b/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java index beaed80f3..46c2f56ca 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java +++ b/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/VectorDataBindingsTest.java @@ -108,10 +108,31 @@ public class VectorDataBindingsTest { } @Test - void aWindowArgumentClipsTheReaderAndWarmsIt() { - BaseVectors clipped = new BaseVectors("example:demo", "1..3", "eager", settings); - assertArrayEquals(new float[] {1f, 2f}, clipped.apply(0), "windowed record 0 is underlying record 1"); - assertThrows(IndexOutOfBoundsException.class, () -> clipped.apply(2), "the window is two records long"); + void aWindowWarmsWithoutRenumbering() { + // The window says which records to warm; it does not re-base + // them. Index 1 is record 1, as it would be with no window. + BaseVectors warmed = new BaseVectors("example:demo", "1..3", "eager", settings); + assertArrayEquals(new float[] {1f, 2f}, warmed.apply(1), "index 1 is record 1, not the window's first record"); + assertArrayEquals(new float[] {2f, 3f}, warmed.apply(2)); + } + + @Test + void aWindowFarFromZeroReadsTheRecordsItsCyclesName() { + // The field failure: a window starting well above zero used to + // re-base to 0, so a run over those same cycles indexed past the + // end and threw. + BaseVectors slice = new BaseVectors("example:demo", "[7..10)", "none", settings); + assertArrayEquals(new float[] {7f, 8f}, slice.apply(7)); + assertArrayEquals(new float[] {9f, 10f}, slice.apply(9)); + } + + @Test + void readingOutsideTheWindowStillWorks() { + // Not an error, and load-bearing: VirtData probes a resolved + // function with a small sample index before any cycle runs, so a + // window starting above zero would otherwise fail to resolve. + BaseVectors slice = new BaseVectors("example:demo", "[7..10)", "none", settings); + assertArrayEquals(new float[] {1f, 2f}, slice.apply(1), "outside the warmed window, but readable"); } @Test @@ -119,16 +140,16 @@ public class VectorDataBindingsTest { BaseVectors vectors = new BaseVectors("example:demo", "1..3", "background", settings); assertNotNull(vectors.backgroundPrefetch(), "background mode hands back the running prefetch"); vectors.backgroundPrefetch().join(); - assertArrayEquals(new float[] {2f, 3f}, vectors.apply(1)); + assertArrayEquals(new float[] {1f, 2f}, vectors.apply(1)); } @Test void aBindingWindowComposesWithAProfileWindow() { - // The profile clips to records [2..8); the binding window is in - // the clipped reader's coordinates, so 1..3 reads records 3..5. + // The profile clips to records [2..8), which re-bases the reader + // it hands back; the binding window then restricts that reader's + // indices 1..2, and warming shifts them to dataset records 3..5. BaseVectors vectors = new BaseVectors("example:windowed", "1..3", "eager", settings); - assertArrayEquals(new float[] {3f, 4f}, vectors.apply(0)); - assertThrows(IndexOutOfBoundsException.class, () -> vectors.apply(2)); + assertArrayEquals(new float[] {3f, 4f}, vectors.apply(1)); // And warming shifts the same window to absolute records 3..5. DSWindow effective = CoreVectors.effectiveWindow(vectors.tdv, "base_vectors", "1..3"); assertEquals(new DSWindow.Interval(3, 5), effective.intervals().get(0)); From 5a82d60305af5cb825bf44afb7954293a2dc4d97 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 2 Sep 2026 20:19:36 +0000 Subject: [PATCH 9/9] fix(expr-api): render datetime seconds even on a whole minute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ISO forms of LocalTime and LocalDateTime omit the seconds field when the second is 00, so currentTime(), currentDateTime(), currentTimeInZone() and millisToDateTime() each returned a shorter string once a minute — HH:mm instead of HH:mm:ss. That failed testCurrentTime in CI roughly one run in sixty, and silently handed workloads a differently-shaped timestamp the rest of the time. Each now formats with an explicit pattern rather than toString(), which also makes the truncatedTo(SECONDS) calls redundant. The new test pins the boundary through millisToDateTime, whose instant comes from the caller: the wall-clock functions can only be asserted for shape, which catches this one second in sixty. It builds a timestamp on a whole minute, requires the seconds to be there, and repeats at second 30 so a fix in the other direction is caught too. --- .../main/resources/lib/groovy/datetime.groovy | 16 ++++++++---- .../nb/api/expr/BundledLibrariesTest.java | 26 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/nb-apis/expr-api/src/main/resources/lib/groovy/datetime.groovy b/nb-apis/expr-api/src/main/resources/lib/groovy/datetime.groovy index 842c033de..0eacfa987 100644 --- a/nb-apis/expr-api/src/main/resources/lib/groovy/datetime.groovy +++ b/nb-apis/expr-api/src/main/resources/lib/groovy/datetime.groovy @@ -76,7 +76,10 @@ def currentDate() { ) @ExprExample(expectNotNull = true) def currentTime() { - LocalTime.now().truncatedTo(ChronoUnit.SECONDS).toString() + // Explicit pattern, not toString(): LocalTime's ISO form omits the + // seconds field entirely when the second is 00, so toString() emits + // HH:mm once a minute and HH:mm:ss the rest of the time. + LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss")) } @ExprFunctionSpec( @@ -85,7 +88,8 @@ def currentTime() { ) @ExprExample(expectNotNull = true) def currentDateTime() { - LocalDateTime.now().truncatedTo(ChronoUnit.SECONDS).toString() + // See currentTime(): the ISO form drops seconds at second 00. + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")) } @ExprFunctionSpec( @@ -266,11 +270,12 @@ def millisToDate(millis) { ) @ExprExample(expectNotNull = true) def millisToDateTime(millis) { + // See currentTime(): the ISO form drops seconds at second 00, so a + // timestamp landing on a whole minute rendered as yyyy-MM-ddTHH:mm. Instant.ofEpochMilli(millis as long) .atZone(ZoneId.systemDefault()) .toLocalDateTime() - .truncatedTo(ChronoUnit.SECONDS) - .toString() + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")) } @ExprFunctionSpec( @@ -300,5 +305,6 @@ def currentDateInZone(timezone) { ) @ExprExample(expectNotNull = true) def currentTimeInZone(timezone) { - LocalTime.now(ZoneId.of(timezone)).truncatedTo(ChronoUnit.SECONDS).toString() + // See currentTime(): the ISO form drops seconds at second 00. + LocalTime.now(ZoneId.of(timezone)).format(DateTimeFormatter.ofPattern("HH:mm:ss")) } diff --git a/nb-apis/expr-api/src/test/java/io/nosqlbench/nb/api/expr/BundledLibrariesTest.java b/nb-apis/expr-api/src/test/java/io/nosqlbench/nb/api/expr/BundledLibrariesTest.java index fdffbcf03..fd65eb13a 100644 --- a/nb-apis/expr-api/src/test/java/io/nosqlbench/nb/api/expr/BundledLibrariesTest.java +++ b/nb-apis/expr-api/src/test/java/io/nosqlbench/nb/api/expr/BundledLibrariesTest.java @@ -19,6 +19,10 @@ package io.nosqlbench.nb.api.expr; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import org.junit.jupiter.api.Tag; import java.util.Map; @@ -122,6 +126,28 @@ class BundledLibrariesTest { assertTrue(result.matches("\\d{2}:\\d{2}:\\d{2}")); } + /// The seconds field is optional in the ISO forms of `LocalTime` and + /// `LocalDateTime`: `toString()` omits it entirely when the second + /// is `00`, so a shape assertion against the wall clock only catches + /// the defect one second in sixty. `millisToDateTime` takes its + /// instant from the caller and shares the same rendering, so the + /// boundary can be pinned exactly rather than waited for. + @Test + void timesOnAWholeMinuteStillRenderTheirSeconds() { + long onTheMinute = ZonedDateTime.now(ZoneId.systemDefault()) + .truncatedTo(ChronoUnit.MINUTES).toInstant().toEpochMilli(); + String atSecondZero = processor.process("{{= millisToDateTime(" + onTheMinute + ") }}", null, Map.of()); + assertTrue(atSecondZero.matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}"), + "a timestamp on a whole minute must still render seconds, got: " + atSecondZero); + assertTrue(atSecondZero.endsWith(":00"), "and those seconds are 00, got: " + atSecondZero); + + String atSecondThirty = processor.process( + "{{= millisToDateTime(" + (onTheMinute + 30_000) + ") }}", null, Map.of()); + assertTrue(atSecondThirty.matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}"), + "and a non-zero second renders the same width, got: " + atSecondThirty); + assertTrue(atSecondThirty.endsWith(":30"), atSecondThirty); + } + @Test void testCurrentDateTime() { String result = processor.process("{{= currentDateTime() }}", null, Map.of());