- New nb-mql-api module with 11 query commands (instant, range, rate, increase, quantile, agg, ratio, topk, session, summary, sql)

- BundledApp integration (@Service selector="mql") accessible via nb5 CLI
- 6 output formats (table, csv, tsv, json, jsonl, markdown) with enum-based architecture
- 107 passing tests with self-validating documentation and 5 test databases
- Add sqlite-jdbc to dependency management and nb-mql-api to nb-engine-all
- Add shutdown hook to update symlink mtimes on session end
This commit is contained in:
Jonathan Shook committed 2025-10-27 19:48:37 +00:00
1 parent 68c35027c5
commit 30c8a9d015
79 files changed
+9825 -2

No files matched your search

+5
View File
@@ -181,6 +181,11 @@
<artifactId>ascii-data</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.46.0.0</version>
</dependency>
<!-- For CQL compression option -->
<dependency>
<groupId>org.lz4</groupId>
-1
View File
@@ -159,7 +159,6 @@
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.46.0.0</version>
</dependency>
</dependencies>
+5
View File
@@ -47,5 +47,10 @@
<artifactId>adapters-api</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-mql-api</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</project>
+139
View File
@@ -0,0 +1,139 @@
<!--
~ Copyright (c) nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>${revision}</version>
<relativePath>../../mvn-defaults</relativePath>
</parent>
<artifactId>nb-mql-api</artifactId>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
<description>
MetricsQL-inspired query engine for NoSQLBench SQLite metrics databases.
Provides read-only query commands for analyzing metrics data with support
for rates, aggregations, quantiles, and time-series analysis.
</description>
<dependencies>
<!-- NoSQLBench API for labels and base types -->
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-api</artifactId>
<version>${revision}</version>
</dependency>
<!-- SQLite JDBC driver (version managed in parent) -->
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
</dependency>
<!-- CLI framework (picocli) - inherited from parent -->
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
</dependency>
<!-- JSON output formatting -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<!-- CSV output formatting -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>
<!-- Logging for standalone JAR (sqlite-jdbc needs SLF4J) -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.16</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
<version>2.25.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.25.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.25.2</version>
</dependency>
<!-- Testing dependencies inherited from parent -->
</dependencies>
<build>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<filtering>false</filtering>
<includes>
<include>**/*.db</include>
</includes>
</testResource>
<testResource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
<excludes>
<exclude>**/*.db</exclude>
</excludes>
</testResource>
</testResources>
<plugins>
<!-- Maven Assembly Plugin for standalone JAR -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<archive>
<manifest>
<mainClass>io.nosqlbench.nb.mql.cli.MetricsQLCLI</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<finalName>nb-metricsql</finalName>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,54 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.apps.BundledApp;
import io.nosqlbench.nb.mql.cli.MetricsQLCLI;
import picocli.CommandLine;
/**
* MetricsQL Query Engine as a NoSQLBench BundledApp.
* Allows querying SQLite metrics databases using MetricsQL-inspired syntax.
*
* Usage:
* nb5 mql instant --metric ops_total
* nb5 mql range --metric ops_total --window 5m
*
* This integrates the MetricsQL CLI as a bundled application within NoSQLBench,
* making it immediately available without needing a separate tool.
*/
@Service(value = BundledApp.class, selector = "mql")
public class MQLApp implements BundledApp {
@Override
public int applyAsInt(String[] args) {
// Delegate to the picocli-based MetricsQLCLI
// This allows all the command routing and validation to work seamlessly
return new CommandLine(new MetricsQLCLI()).execute(args);
}
/**
* Standalone main method for direct execution (optional).
* Can also be run via: java -cp ... io.nosqlbench.nb.mql.MQLApp instant --metric ops
*/
public static void main(String[] args) {
MQLApp app = new MQLApp();
int exitCode = app.applyAsInt(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,124 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.format.OutputFormat;
import io.nosqlbench.nb.mql.format.ResultFormatter;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import picocli.CommandLine.Option;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.Callable;
/**
* Abstract base class for all query command CLIs.
* Provides common functionality for database connection, formatting, and error handling.
*
* Subclasses must implement:
* - createCommand() - Create the specific MetricsQueryCommand instance
* - buildParams() - Build the parameter map for the command
*/
public abstract class AbstractQueryCommandCLI implements Callable<Integer> {
@Option(
names = {"-d", "--db"},
description = "Path to metrics database (default: logs/metrics.db)",
defaultValue = "logs/metrics.db"
)
protected Path databasePath;
@Option(
names = {"-m", "--metric"},
description = "Metric name to query",
required = true
)
protected String metric;
@Option(
names = {"-l", "--labels"},
description = "Label filters as key=value,key=value",
split = ","
)
protected Map<String, String> labels = new LinkedHashMap<>();
@Option(
names = {"-f", "--format"},
description = "Output format: table, json, jsonl, csv, tsv, markdown",
defaultValue = "table"
)
protected OutputFormat format = OutputFormat.TABLE;
@Option(
names = {"--explain"},
description = "Show the SQL query instead of executing"
)
protected boolean explain;
@Override
public Integer call() throws Exception {
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
MetricsQueryCommand command = createCommand();
Map<String, Object> params = buildParams();
QueryResult result = command.execute(conn, params);
if (explain) {
System.out.println("SQL Query:");
System.out.println(result.executedSQL());
return 0;
}
ResultFormatter formatter = format.createFormatter();
System.out.println(formatter.format(result));
return 0;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
if (System.getenv("DEBUG") != null) {
e.printStackTrace();
}
return 1;
}
}
/**
* Create the specific command instance for this CLI.
*
* @return MetricsQueryCommand instance
*/
protected abstract MetricsQueryCommand createCommand();
/**
* Build the parameter map for the command.
* Subclasses override to add command-specific parameters.
*
* @return Parameter map
*/
protected Map<String, Object> buildParams() {
Map<String, Object> params = new LinkedHashMap<>();
params.put("metric", metric);
if (!labels.isEmpty()) {
params.put("labels", labels);
}
return params;
}
}
@@ -0,0 +1,97 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.AggregateCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* CLI for aggregate query command.
* Performs aggregations (sum, avg, max, min, count) with optional grouping.
*/
@Command(
name = "agg",
description = "Perform aggregations (sum, avg, max, min, count) on metrics",
mixinStandardHelpOptions = true
)
public class AggregateCLI extends AbstractQueryCommandCLI {
@Option(
names = {"--func", "--function"},
description = "Aggregate function: sum, avg, max, min, count",
required = true
)
private String function;
@Option(
names = {"--by"},
description = "Group by labels (comma-separated, e.g., activity,host)",
split = ","
)
private List<String> groupBy;
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h) - optional, defaults to latest snapshot"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window"
)
private String last;
@Override
protected MetricsQueryCommand createCommand() {
return new AggregateCommand();
}
@Override
protected Map<String, Object> buildParams() {
Map<String, Object> params = super.buildParams();
params.put("function", function);
if (groupBy != null && !groupBy.isEmpty()) {
params.put("by", groupBy);
}
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
}
return params;
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new AggregateCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,76 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.IncreaseCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.util.Map;
/**
* CLI for increase query command.
* Calculates total increase in counter value over a time window.
*/
@Command(
name = "increase",
description = "Calculate total increase in counter over time window",
mixinStandardHelpOptions = true
)
public class IncreaseCLI extends AbstractQueryCommandCLI {
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h, 30s)"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window (e.g., last 5m)"
)
private String last;
@Override
protected MetricsQueryCommand createCommand() {
return new IncreaseCommand();
}
@Override
protected Map<String, Object> buildParams() {
Map<String, Object> params = super.buildParams();
// Add time window parameter
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
}
return params;
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new IncreaseCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,55 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.InstantCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import java.util.Map;
/**
* CLI for instant query command.
* Can be run standalone: java -cp ... io.nosqlbench.nb.mql.cli.InstantCLI --metric ops_total
*/
@Command(
name = "instant",
description = "Get the most recent value for a metric",
mixinStandardHelpOptions = true
)
public class InstantCLI extends AbstractQueryCommandCLI {
@Override
protected MetricsQueryCommand createCommand() {
return new InstantCommand();
}
@Override
protected Map<String, Object> buildParams() {
// Use base class params (metric and labels)
return super.buildParams();
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new InstantCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,68 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import picocli.CommandLine;
import picocli.CommandLine.Command;
/**
* Main entry point for the MetricsQL CLI tool.
* Routes to subcommands for different query types.
*
* Usage:
* nb-metricsql instant --metric ops_total
* nb-metricsql range --metric ops_total --window 5m
* nb-metricsql rate --metric ops_total --window 5m
* ...
*/
@Command(
name = "nb-metricsql",
description = "MetricsQL query engine for NoSQLBench SQLite metrics databases",
mixinStandardHelpOptions = true,
version = "1.0.0",
subcommands = {
SessionCLI.class,
SummaryCLI.class,
InstantCLI.class,
RangeCLI.class,
RateCLI.class,
IncreaseCLI.class,
QuantileCLI.class,
AggregateCLI.class,
RatioCLI.class,
TopKCLI.class,
SqlCLI.class
// All commands implemented except:
// BinaryOpCLI.class (optional - advanced arithmetic)
}
)
public class MetricsQLCLI implements Runnable {
@Override
public void run() {
// If no subcommand specified, show help
CommandLine.usage(this, System.out);
}
/**
* Main entry point for the standalone JAR.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new MetricsQLCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,85 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.QuantileCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.util.Map;
/**
* CLI for quantile query command.
* Extracts percentile values from timer/summary metrics.
*/
@Command(
name = "quantile",
description = "Extract percentile values from timer/summary metrics",
mixinStandardHelpOptions = true
)
public class QuantileCLI extends AbstractQueryCommandCLI {
@Option(
names = {"-p", "--percentile", "--quantile"},
description = "Quantile value (0.0-1.0, e.g., 0.95 for p95)",
required = true
)
private double quantile;
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h) - optional, defaults to all data"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window (e.g., last 5m)"
)
private String last;
@Override
protected MetricsQueryCommand createCommand() {
return new QuantileCommand();
}
@Override
protected Map<String, Object> buildParams() {
Map<String, Object> params = super.buildParams();
params.put("quantile", quantile);
// Add time window if specified
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
}
return params;
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new QuantileCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,91 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.RangeCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.util.Map;
/**
* CLI for range query command.
* Can be run standalone: java -cp ... io.nosqlbench.nb.mql.cli.RangeCLI --metric ops_total --window 5m
*/
@Command(
name = "range",
description = "Retrieve time-series data over a time window",
mixinStandardHelpOptions = true
)
public class RangeCLI extends AbstractQueryCommandCLI {
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h, 30s)"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window (e.g., last 5m)"
)
private String last;
@Option(
names = {"--start"},
description = "Start timestamp in epoch milliseconds"
)
private Long start;
@Option(
names = {"--end"},
description = "End timestamp in epoch milliseconds"
)
private Long end;
@Override
protected MetricsQueryCommand createCommand() {
return new RangeCommand();
}
@Override
protected Map<String, Object> buildParams() {
Map<String, Object> params = super.buildParams();
// Add time range parameters
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
} else if (start != null && end != null) {
params.put("start", start);
params.put("end", end);
}
return params;
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new RangeCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,76 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.RateCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.util.Map;
/**
* CLI for rate query command.
* Calculates per-second rate of change for counter metrics.
*/
@Command(
name = "rate",
description = "Calculate per-second rate of change for counters",
mixinStandardHelpOptions = true
)
public class RateCLI extends AbstractQueryCommandCLI {
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h, 30s)"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window (e.g., last 5m)"
)
private String last;
@Override
protected MetricsQueryCommand createCommand() {
return new RateCommand();
}
@Override
protected Map<String, Object> buildParams() {
Map<String, Object> params = super.buildParams();
// Add time window parameter
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
}
return params;
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new RateCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,145 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.RatioCommand;
import io.nosqlbench.nb.mql.format.OutputFormat;
import io.nosqlbench.nb.mql.format.ResultFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.Callable;
/**
* CLI for ratio query command.
* Calculates ratio between two metrics (useful for error rates).
*/
@Command(
name = "ratio",
description = "Calculate ratio between two metrics (numerator / denominator)",
mixinStandardHelpOptions = true
)
public class RatioCLI implements Callable<Integer> {
@Option(
names = {"-d", "--db"},
description = "Path to metrics database (default: logs/metrics.db)",
defaultValue = "logs/metrics.db"
)
private Path databasePath;
@Option(
names = {"-n", "--numerator"},
description = "Numerator metric name",
required = true
)
private String numerator;
@Option(
names = {"--denominator"},
description = "Denominator metric name",
required = true
)
private String denominator;
@Option(
names = {"-l", "--labels"},
description = "Label filters (applied to both metrics)",
split = ","
)
private Map<String, String> labels = new LinkedHashMap<>();
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h)"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window"
)
private String last;
@Option(
names = {"-f", "--format"},
description = "Output format: table, json, jsonl, csv, tsv, markdown",
defaultValue = "table"
)
private OutputFormat format = OutputFormat.TABLE;
@Option(
names = {"--explain"},
description = "Show the SQL query instead of executing"
)
private boolean explain;
@Override
public Integer call() throws Exception {
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
RatioCommand command = new RatioCommand();
Map<String, Object> params = new LinkedHashMap<>();
params.put("numerator", numerator);
params.put("denominator", denominator);
if (!labels.isEmpty()) {
params.put("labels", labels);
}
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
}
QueryResult result = command.execute(conn, params);
if (explain) {
System.out.println("SQL Query:");
System.out.println(result.executedSQL());
return 0;
}
ResultFormatter formatter = format.createFormatter();
System.out.println(formatter.format(result));
return 0;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
if (System.getenv("DEBUG") != null) {
e.printStackTrace();
}
return 1;
}
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new RatioCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,84 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.SessionCommand;
import io.nosqlbench.nb.mql.format.OutputFormat;
import io.nosqlbench.nb.mql.format.ResultFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import java.util.concurrent.Callable;
/**
* CLI for session command.
* Displays session time information.
*/
@Command(
name = "session",
description = "Display session time information and snapshot cadence",
mixinStandardHelpOptions = true
)
public class SessionCLI implements Callable<Integer> {
@Option(
names = {"-d", "--db"},
description = "Path to metrics database (default: logs/metrics.db)",
defaultValue = "logs/metrics.db"
)
private Path databasePath;
@Option(
names = {"-f", "--format"},
description = "Output format: table, json, jsonl, csv, tsv, markdown",
defaultValue = "table"
)
private OutputFormat format = OutputFormat.TABLE;
@Override
public Integer call() throws Exception {
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
SessionCommand command = new SessionCommand();
QueryResult result = command.execute(conn, Map.of());
ResultFormatter formatter = format.createFormatter();
System.out.println(formatter.format(result));
return 0;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
if (System.getenv("DEBUG") != null) {
e.printStackTrace();
}
return 1;
}
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new SessionCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,104 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.SqlCommand;
import io.nosqlbench.nb.mql.format.OutputFormat;
import io.nosqlbench.nb.mql.format.ResultFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.Callable;
/**
* CLI for raw SQL query execution.
* Allows direct SQLite queries for advanced users.
*/
@Command(
name = "sql",
description = "Execute raw SQLite queries (read-only)",
mixinStandardHelpOptions = true
)
public class SqlCLI implements Callable<Integer> {
@Option(
names = {"-d", "--db"},
description = "Path to metrics database (default: logs/metrics.db)",
defaultValue = "logs/metrics.db"
)
private Path databasePath;
@Option(
names = {"-q", "--query"},
description = "SQL query to execute (SELECT, WITH, or PRAGMA only)",
required = true
)
private String query;
@Option(
names = {"-t", "--timeout"},
description = "Query timeout in seconds (default: 30)",
defaultValue = "30"
)
private int timeout;
@Option(
names = {"-f", "--format"},
description = "Output format: table, json, jsonl, csv, tsv, markdown",
defaultValue = "table"
)
private OutputFormat format = OutputFormat.TABLE;
@Override
public Integer call() throws Exception {
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
SqlCommand command = new SqlCommand();
Map<String, Object> params = new LinkedHashMap<>();
params.put("query", query);
params.put("timeout", timeout);
QueryResult result = command.execute(conn, params);
ResultFormatter formatter = format.createFormatter();
System.out.println(formatter.format(result));
return 0;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
if (System.getenv("DEBUG") != null) {
e.printStackTrace();
}
return 1;
}
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new SqlCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,92 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.SummaryCommand;
import io.nosqlbench.nb.mql.format.OutputFormat;
import io.nosqlbench.nb.mql.format.ResultFormatter;
import io.nosqlbench.nb.mql.format.SummaryMarkdownFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import java.util.concurrent.Callable;
/**
* CLI for summary command.
* Displays database overview and metrics inventory.
*/
@Command(
name = "summary",
description = "Display overview of metrics in the database",
mixinStandardHelpOptions = true
)
public class SummaryCLI implements Callable<Integer> {
@Option(
names = {"-d", "--db"},
description = "Path to metrics database (default: logs/metrics.db)",
defaultValue = "logs/metrics.db"
)
private Path databasePath;
@Option(
names = {"-f", "--format"},
description = "Output format: table, json, jsonl, csv, tsv, markdown",
defaultValue = "table"
)
private OutputFormat format = OutputFormat.TABLE;
@Override
public Integer call() throws Exception {
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
SummaryCommand command = new SummaryCommand();
QueryResult result = command.execute(conn, Map.of());
// Use custom markdown formatter for summary command
ResultFormatter formatter;
if (format == OutputFormat.MARKDOWN) {
formatter = new SummaryMarkdownFormatter();
} else {
formatter = format.createFormatter();
}
System.out.println(formatter.format(result));
return 0;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
if (System.getenv("DEBUG") != null) {
e.printStackTrace();
}
return 1;
}
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new SummaryCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,84 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.cli;
import io.nosqlbench.nb.mql.commands.TopKCommand;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import java.util.Map;
/**
* CLI for topk query command.
* Finds top N label sets by metric value.
*/
@Command(
name = "topk",
description = "Find top N label sets by metric value",
mixinStandardHelpOptions = true
)
public class TopKCLI extends AbstractQueryCommandCLI {
@Option(
names = {"-n", "--top"},
description = "Number of top results to return",
required = true
)
private int n;
@Option(
names = {"-w", "--window"},
description = "Time window (e.g., 5m, 1h) - optional, defaults to latest snapshot"
)
private String window;
@Option(
names = {"--last"},
description = "Alternative to --window"
)
private String last;
@Override
protected MetricsQueryCommand createCommand() {
return new TopKCommand();
}
@Override
protected Map<String, Object> buildParams() {
Map<String, Object> params = super.buildParams();
params.put("n", n);
if (window != null) {
params.put("window", window);
} else if (last != null) {
params.put("last", last);
}
return params;
}
/**
* Main method for standalone execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new TopKCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,318 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.AggregateCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
/**
* Aggregate command - Perform aggregations (sum, avg, max, min, count) on metrics.
*
* Usage: agg --metric <name> --func <sum|avg|max|min|count> [--by <label,...>] [--window <duration>] [--labels key=val,...]
*
* Aggregates metric values using SQL aggregate functions, optionally grouping by label dimensions.
* This is the PromQL aggregation operator (sum by, avg by, etc.).
*/
public class AggregateCommand implements MetricsQueryCommand {
private static final Set<String> VALID_FUNCTIONS = Set.of("sum", "avg", "max", "min", "count");
@Override
public String getName() {
return "agg";
}
@Override
public String getDescription() {
return "Perform aggregations (sum, avg, max, min, count) on metrics";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
String function = ((String) params.get("function")).toLowerCase();
@SuppressWarnings("unchecked")
List<String> groupByLabels = (List<String>) params.getOrDefault("by", List.of());
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range (optional - defaults to latest snapshot)
Long startTimestampMs = null;
Long endTimestampMs = null;
boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last");
if (params.containsKey("window")) {
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("last")) {
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
// Build the query
String sql = buildQuery(metric, function, groupByLabels, labels, startTimestampMs, endTimestampMs, useLatestOnly);
// Build columns dynamically based on grouping
List<String> columns = new ArrayList<>();
columns.addAll(groupByLabels);
columns.add(function);
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
if (!useLatestOnly && startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
// Add grouped label values
for (String labelName : groupByLabels) {
row.put(labelName, rs.getString(labelName));
}
// Add aggregate result
row.put(function, rs.getDouble(function));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
if (!params.containsKey("function")) {
throw new InvalidQueryException("Missing required parameter: function (sum, avg, max, min, count)");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
Object function = params.get("function");
if (!(function instanceof String)) {
throw new InvalidQueryException("Parameter 'function' must be a string");
}
String func = ((String) function).toLowerCase();
if (!VALID_FUNCTIONS.contains(func)) {
throw new InvalidQueryException(
"Invalid function: " + function + "\n" +
"Valid functions: " + String.join(", ", VALID_FUNCTIONS)
);
}
// Validate 'by' parameter if present
if (params.containsKey("by")) {
Object by = params.get("by");
if (!(by instanceof List)) {
throw new InvalidQueryException("Parameter 'by' must be a List<String>");
}
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Sum all requests
agg --metric api_requests_total --func sum
# Average by endpoint
agg --metric api_requests_total --func avg --by endpoint
# Max latency by operation
agg --metric operation_latency --func max --by operation
# Count by status code and method
agg --metric api_requests_total --func count --by status,method
# Sum over last 5 minutes
agg --metric requests_total --func sum --window 5m
""";
}
private String buildQuery(String metric, String function, List<String> groupByLabels,
Map<String, String> labelFilters, Long startTs, Long endTs, boolean useLatestOnly) {
StringBuilder sql = new StringBuilder();
sql.append("WITH labeled_values AS (\n");
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE);
// Add individual label columns for grouping
if (!groupByLabels.isEmpty()) {
sql.append(",\n");
for (int i = 0; i < groupByLabels.size(); i++) {
sql.append(" lk").append(i).append(".").append(MetricsSchema.COL_LK_NAME).append(" AS label_key_").append(i).append(",\n");
sql.append(" lv").append(i).append(".").append(MetricsSchema.COL_LV_VALUE).append(" AS label_value_").append(i);
if (i < groupByLabels.size() - 1) {
sql.append(",\n");
}
}
}
sql.append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
if (useLatestOnly) {
sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ")
.append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n");
}
// Join with label tables for grouping
for (int i = 0; i < groupByLabels.size(); i++) {
String labelName = groupByLabels.get(i);
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm").append(i)
.append(" ON lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_SET_ID)
.append(" = sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk").append(i)
.append(" ON lk").append(i).append(".").append(MetricsSchema.COL_LK_ID)
.append(" = lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_KEY_ID)
.append(" AND lk").append(i).append(".").append(MetricsSchema.COL_LK_NAME).append(" = '").append(labelName).append("'\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv").append(i)
.append(" ON lv").append(i).append(".").append(MetricsSchema.COL_LV_ID)
.append(" = lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
}
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
if (useLatestOnly) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n");
} else if (startTs != null) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
int filterIdx = 0;
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
filterIdx++;
}
sql.append(")\n");
// Aggregation query
sql.append("SELECT\n");
for (int i = 0; i < groupByLabels.size(); i++) {
sql.append(" label_value_").append(i).append(" AS ").append(groupByLabels.get(i)).append(",\n");
}
sql.append(" ").append(function.toUpperCase()).append("(value) AS ").append(function).append("\n");
sql.append("FROM labeled_values\n");
if (!groupByLabels.isEmpty()) {
sql.append("GROUP BY ");
for (int i = 0; i < groupByLabels.size(); i++) {
sql.append("label_value_").append(i);
if (i < groupByLabels.size() - 1) {
sql.append(", ");
}
}
sql.append("\n");
}
sql.append("ORDER BY ").append(function);
if (function.equals("count") || function.equals("sum")) {
sql.append(" DESC"); // Descending for counts and sums
}
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new AggregateCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,249 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.IncreaseCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Increase command - Calculate total increase in counter value over a time window.
*
* Usage: increase --metric <counter_metric> --window <duration> [--labels key=val,...]
*
* Returns the total increase (max - min) for each label set within the time window.
* This is the standard PromQL increase() function.
*
* Important: Works only with counter metrics (monotonically increasing values).
*/
public class IncreaseCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "increase";
}
@Override
public String getDescription() {
return "Calculate total increase in counter over time window";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range
Long startTimestampMs = null;
Long endTimestampMs = null;
if (params.containsKey("window")) {
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("last")) {
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
// Build the query
String sql = buildQuery(labels, startTimestampMs != null);
List<String> columns = new ArrayList<>();
columns.add("increase");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
if (startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("increase", rs.getDouble("increase"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
// Check that either window OR last is provided
boolean hasWindow = params.containsKey("window");
boolean hasLast = params.containsKey("last");
if (!hasWindow && !hasLast) {
throw new InvalidQueryException(
"Must specify time window using:\n" +
" --window <duration> (e.g., 5m, 1h)\n" +
" --last <duration> (e.g., 5m, 1h)"
);
}
if (hasWindow && hasLast) {
throw new InvalidQueryException(
"Cannot specify both --window and --last. Choose one."
);
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Total requests in last hour
increase --metric api_requests_total --window 1h
# Total errors in last 5 minutes
increase --metric api_requests_total --window 5m --labels status=500
# Operations completed in last day
increase --metric activity_ops_total --window 1d
""";
}
private String buildQuery(Map<String, String> labelFilters, boolean hasTimeRange) {
StringBuilder sql = new StringBuilder();
sql.append("WITH timeseries AS (\n");
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
if (hasTimeRange) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
sql.append(")\n");
// Calculate increase as MAX - MIN within window for each label set
sql.append("SELECT\n");
sql.append(" MAX(value) - MIN(value) AS increase,\n");
sql.append(" labels\n");
sql.append("FROM timeseries\n");
sql.append("GROUP BY label_set_id, labels\n");
sql.append("HAVING MAX(value) > MIN(value)\n"); // Only show increases > 0
sql.append("ORDER BY labels");
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution of increase queries.
* Delegates to IncreaseCLI for picocli argument parsing.
*
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.IncreaseCommand --metric ops_total --window 1h
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new IncreaseCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,192 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.InstantCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Instant query command - retrieves the most recent value for metrics.
*
* Usage: instant --metric <metric_name> [--labels key=val,...]
*
* Returns the latest snapshot value for each unique label set.
*/
public class InstantCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "instant";
}
@Override
public String getDescription() {
return "Get the most recent value for a metric";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Build the query
String sql = buildQuery(labels);
List<String> columns = new ArrayList<>();
columns.add("timestamp");
columns.add("value");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("timestamp", rs.getTimestamp("timestamp"));
row.put("value", rs.getDouble("value"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Get latest value for all label sets
instant --metric activity_ops_total
# Get latest value for specific labels
instant --metric activity_ops_total --labels activity=read,host=server1
# Get latest latency
instant --metric operation_latency_ms
""";
}
private String buildQuery(Map<String, String> labelFilters) {
StringBuilder sql = new StringBuilder();
sql.append("WITH latest_snapshot AS (\n");
sql.append(" SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append("\n");
sql.append("),\n");
sql.append("labeled_samples AS (\n");
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME).append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels,\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.").append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
sql.append(" CROSS JOIN latest_snapshot\n");
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest_snapshot.max_ts\n");
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.").append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.").append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.").append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
sql.append(")\n");
sql.append("SELECT\n");
sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n");
sql.append(" value,\n");
sql.append(" labels\n");
sql.append("FROM labeled_samples\n");
sql.append("ORDER BY labels");
return sql.toString();
}
/**
* Standalone main method for direct execution of instant queries.
* Delegates to InstantCLI for picocli argument parsing.
*
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.InstantCommand --metric ops_total
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new InstantCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,250 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.QuantileCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Quantile command - Extract percentile values from timer/summary metrics.
*
* Usage: quantile --metric <timer_metric> --p <0.0-1.0> [--window <duration>] [--labels key=val,...]
*
* Retrieves specific quantile values (e.g., p50, p95, p99) from timer or summary metrics.
* Useful for latency analysis and SLA monitoring.
*/
public class QuantileCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "quantile";
}
@Override
public String getDescription() {
return "Extract percentile values from timer/summary metrics";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
Double quantile = (Double) params.get("quantile");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range (optional for quantile)
Long startTimestampMs = null;
Long endTimestampMs = null;
if (params.containsKey("window")) {
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("last")) {
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
// Build the query
String sql = buildQuery(labels, quantile, startTimestampMs != null);
List<String> columns = new ArrayList<>();
columns.add("timestamp");
columns.add("quantile");
columns.add("quantile_value");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
ps.setDouble(paramIndex++, quantile);
if (startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("timestamp", rs.getTimestamp("timestamp"));
row.put("quantile", rs.getDouble("quantile"));
row.put("quantile_value", rs.getDouble("quantile_value"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
if (!params.containsKey("quantile")) {
throw new InvalidQueryException("Missing required parameter: quantile (e.g., 0.95 for p95)");
}
Object quantile = params.get("quantile");
if (!(quantile instanceof Double)) {
throw new InvalidQueryException("Parameter 'quantile' must be a number between 0.0 and 1.0");
}
double q = (Double) quantile;
if (q < 0.0 || q > 1.0) {
throw new InvalidQueryException("Parameter 'quantile' must be between 0.0 and 1.0, got: " + q);
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Get p95 latency for all operations
quantile --metric operation_latency_ms --p 0.95
# Get p99 for specific operation
quantile --metric api_latency_ms --p 0.99 --labels operation=select
# p50 (median) over last 5 minutes
quantile --metric latency_ms --p 0.50 --window 5m
# Common percentiles
--p 0.50 # p50 (median)
--p 0.95 # p95
--p 0.99 # p99
--p 0.999 # p999
""";
}
private String buildQuery(Map<String, String> labelFilters, double quantile, boolean hasTimeRange) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT\n");
sql.append(" datetime(sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" / 1000, 'unixepoch') AS timestamp,\n");
sql.append(" sq.").append(MetricsSchema.COL_SQ_QUANTILE).append(",\n");
sql.append(" sq.").append(MetricsSchema.COL_SQ_QUANTILE_VALUE).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
sql.append("FROM ").append(MetricsSchema.TABLE_SAMPLE_QUANTILE).append(" sq\n");
sql.append("JOIN ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv ON sq.")
.append(MetricsSchema.COL_SQ_SAMPLE_VALUE_ID).append(" = sv.").append(MetricsSchema.COL_SV_ID).append("\n");
sql.append("JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sv.")
.append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append(" = sn.").append(MetricsSchema.COL_SN_ID).append("\n");
sql.append("").append(MetricsSchema.joinAllLabels()).append("\n");
sql.append("WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
sql.append(" AND sq.").append(MetricsSchema.COL_SQ_QUANTILE).append(" = ?\n");
if (hasTimeRange) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append("GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append(", sq.")
.append(MetricsSchema.COL_SQ_QUANTILE).append(", sq.").append(MetricsSchema.COL_SQ_QUANTILE_VALUE).append("\n");
sql.append("ORDER BY sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(", labels");
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution of quantile queries.
* Delegates to QuantileCLI for picocli argument parsing.
*
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.QuantileCommand --metric latency_ms --p 0.95
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new QuantileCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,259 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.RangeCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Range query command - retrieves time-series data over a time window.
*
* Usage: range --metric <metric_name> --window <duration> [--labels key=val,...]
* range --metric <metric_name> --start <timestamp> --end <timestamp> [--labels key=val,...]
*
* Returns all data points within the specified time range for each label set.
*/
public class RangeCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "range";
}
@Override
public String getDescription() {
return "Retrieve time-series data over a time window";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range
Long startTimestampMs = null;
Long endTimestampMs = null;
if (params.containsKey("window")) {
// Use --window to query last N duration from now
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("start") && params.containsKey("end")) {
// Use explicit start/end timestamps (epoch millis)
startTimestampMs = (Long) params.get("start");
endTimestampMs = (Long) params.get("end");
} else if (params.containsKey("last")) {
// Alternative: --last 5m
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
// Build the query
String sql = buildQuery(labels, startTimestampMs != null);
List<String> columns = new ArrayList<>();
columns.add("timestamp");
columns.add("value");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
if (startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("timestamp", rs.getTimestamp("timestamp"));
row.put("value", rs.getDouble("value"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
// Check that either window OR (start AND end) OR last is provided
boolean hasWindow = params.containsKey("window");
boolean hasStartEnd = params.containsKey("start") && params.containsKey("end");
boolean hasLast = params.containsKey("last");
if (!hasWindow && !hasStartEnd && !hasLast) {
throw new InvalidQueryException(
"Must specify time range using one of:\n" +
" --window <duration> (e.g., 5m, 1h)\n" +
" --last <duration> (e.g., 5m, 1h)\n" +
" --start <ms> --end <ms> (epoch milliseconds)"
);
}
if ((hasWindow && hasStartEnd) || (hasWindow && hasLast) || (hasStartEnd && hasLast)) {
throw new InvalidQueryException(
"Cannot specify multiple time range methods. Choose one of: --window, --last, or --start/--end"
);
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Get last 5 minutes of data
range --metric activity_ops_total --window 5m
# Alternative syntax
range --metric activity_ops_total --last 5m
# Get data between specific timestamps
range --metric activity_ops_total --start 1729681200000 --end 1729681500000
# With label filters
range --metric activity_ops_total --window 10m --labels activity=read
""";
}
private String buildQuery(Map<String, String> labelFilters, boolean hasTimeRange) {
StringBuilder sql = new StringBuilder();
sql.append("WITH labeled_samples AS (\n");
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels,\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
if (hasTimeRange) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
sql.append(")\n");
sql.append("SELECT\n");
sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n");
sql.append(" value,\n");
sql.append(" labels\n");
sql.append("FROM labeled_samples\n");
sql.append("ORDER BY timestamp_ms, labels");
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution of range queries.
* Delegates to RangeCLI for picocli argument parsing.
*
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.RangeCommand --metric ops_total --window 5m
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new RangeCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,273 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.RateCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Rate command - Calculate per-second rate of change for counter metrics.
*
* Usage: rate --metric <counter_metric> --window <duration> [--labels key=val,...]
*
* Calculates the per-second rate by taking the difference between consecutive data points
* and dividing by the time interval between them. This is the standard PromQL rate() function.
*
* Important: Works only with counter metrics (monotonically increasing values).
*/
public class RateCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "rate";
}
@Override
public String getDescription() {
return "Calculate per-second rate of change for counters";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range
Long startTimestampMs = null;
Long endTimestampMs = null;
if (params.containsKey("window")) {
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("last")) {
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
// Build the query
String sql = buildQuery(labels, startTimestampMs != null);
List<String> columns = new ArrayList<>();
columns.add("timestamp");
columns.add("rate_per_sec");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
if (startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("timestamp", rs.getTimestamp("timestamp"));
row.put("rate_per_sec", rs.getDouble("rate_per_sec"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
// Check that either window OR last is provided (rate needs a time context)
boolean hasWindow = params.containsKey("window");
boolean hasLast = params.containsKey("last");
if (!hasWindow && !hasLast) {
throw new InvalidQueryException(
"Must specify time window using:\n" +
" --window <duration> (e.g., 5m, 1h)\n" +
" --last <duration> (e.g., 5m, 1h)"
);
}
if (hasWindow && hasLast) {
throw new InvalidQueryException(
"Cannot specify both --window and --last. Choose one."
);
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Calculate request rate over last 5 minutes
rate --metric api_requests_total --window 5m
# Alternative syntax
rate --metric api_requests_total --last 5m
# With label filters
rate --metric api_requests_total --window 5m --labels method=GET,status=200
# Longer window for smoother rate
rate --metric activity_ops_total --window 1h
""";
}
private String buildQuery(Map<String, String> labelFilters, boolean hasTimeRange) {
StringBuilder sql = new StringBuilder();
// Use window functions to calculate rate between consecutive points
sql.append("WITH timeseries AS (\n");
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
if (hasTimeRange) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
sql.append(" ORDER BY sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(", sv.")
.append(MetricsSchema.COL_SV_TIMESTAMP_MS).append("\n");
sql.append("),\n");
// Calculate rates using LAG window function
sql.append("rates AS (\n");
sql.append(" SELECT\n");
sql.append(" timestamp_ms,\n");
sql.append(" label_set_id,\n");
sql.append(" labels,\n");
sql.append(" value,\n");
sql.append(" LAG(value) OVER (PARTITION BY label_set_id ORDER BY timestamp_ms) AS prev_value,\n");
sql.append(" LAG(timestamp_ms) OVER (PARTITION BY label_set_id ORDER BY timestamp_ms) AS prev_timestamp\n");
sql.append(" FROM timeseries\n");
sql.append(")\n");
sql.append("SELECT\n");
sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n");
sql.append(" CASE\n");
sql.append(" WHEN prev_value IS NULL OR prev_timestamp IS NULL THEN NULL\n");
sql.append(" WHEN timestamp_ms = prev_timestamp THEN 0.0\n");
sql.append(" ELSE (value - prev_value) / ((timestamp_ms - prev_timestamp) / 1000.0)\n");
sql.append(" END AS rate_per_sec,\n");
sql.append(" labels\n");
sql.append("FROM rates\n");
sql.append("WHERE prev_value IS NOT NULL\n"); // Skip first point (no previous value)
sql.append("ORDER BY timestamp_ms, labels");
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution of rate queries.
* Delegates to RateCLI for picocli argument parsing.
*
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.RateCommand --metric ops_total --window 5m
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new RateCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,291 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.RatioCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Ratio command - Calculate ratio between two metrics (e.g., error rate).
*
* Usage: ratio --numerator <metric> --denominator <metric> [--window <duration>] [--labels key=val,...]
*
* Divides numerator metric by denominator metric for matched label sets.
* Useful for calculating error rates, cache hit ratios, etc.
*/
public class RatioCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "ratio";
}
@Override
public String getDescription() {
return "Calculate ratio between two metrics (numerator / denominator)";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String numerator = (String) params.get("numerator");
String denominator = (String) params.get("denominator");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range (optional - defaults to latest)
Long startTimestampMs = null;
Long endTimestampMs = null;
boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last");
if (params.containsKey("window")) {
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("last")) {
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
String sql = buildQuery(numerator, denominator, labels, startTimestampMs, endTimestampMs, useLatestOnly);
List<String> columns = new ArrayList<>();
if (!useLatestOnly) {
columns.add("timestamp");
}
columns.add("ratio");
columns.add("numerator_value");
columns.add("denominator_value");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, numerator);
ps.setString(paramIndex++, denominator);
if (!useLatestOnly && startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters (twice - once for each metric)
for (int i = 0; i < 2; i++) {
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
if (!useLatestOnly) {
row.put("timestamp", rs.getTimestamp("timestamp"));
}
row.put("ratio", rs.getDouble("ratio"));
row.put("numerator_value", rs.getDouble("numerator_value"));
row.put("denominator_value", rs.getDouble("denominator_value"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("numerator")) {
throw new InvalidQueryException("Missing required parameter: numerator");
}
if (!params.containsKey("denominator")) {
throw new InvalidQueryException("Missing required parameter: denominator");
}
Object numerator = params.get("numerator");
if (!(numerator instanceof String) || ((String) numerator).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'numerator' must be a non-empty string");
}
Object denominator = params.get("denominator");
if (!(denominator instanceof String) || ((String) denominator).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'denominator' must be a non-empty string");
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Error rate (errors / total requests)
ratio --numerator errors_total --denominator requests_total
# Cache hit rate
ratio --numerator cache_hits --denominator cache_total
# Success rate over time
ratio --numerator success_requests --denominator total_requests --window 5m
# With label filters
ratio --numerator errors --denominator requests --labels endpoint=/api/users
""";
}
private String buildQuery(String numerator, String denominator, Map<String, String> labelFilters,
Long startTs, Long endTs, boolean useLatestOnly) {
StringBuilder sql = new StringBuilder();
// Build CTEs for numerator and denominator
sql.append("WITH num_data AS (\n");
sql.append(buildMetricCTE(numerator, "num", labelFilters, startTs, endTs, useLatestOnly));
sql.append("),\n");
sql.append("den_data AS (\n");
sql.append(buildMetricCTE(denominator, "den", labelFilters, startTs, endTs, useLatestOnly));
sql.append(")\n");
// Join and calculate ratio
sql.append("SELECT\n");
if (!useLatestOnly) {
sql.append(" num_data.timestamp,\n");
}
sql.append(" CASE\n");
sql.append(" WHEN den_data.value = 0 THEN NULL\n");
sql.append(" ELSE num_data.value / den_data.value\n");
sql.append(" END AS ratio,\n");
sql.append(" num_data.value AS numerator_value,\n");
sql.append(" den_data.value AS denominator_value,\n");
sql.append(" num_data.labels\n");
sql.append("FROM num_data\n");
sql.append("JOIN den_data ON num_data.label_set_id = den_data.label_set_id");
if (!useLatestOnly) {
sql.append(" AND num_data.timestamp_ms = den_data.timestamp_ms");
}
sql.append("\nORDER BY ");
if (!useLatestOnly) {
sql.append("num_data.timestamp_ms, ");
}
sql.append("num_data.labels");
return sql.toString();
}
private String buildMetricCTE(String metric, String alias, Map<String, String> labelFilters,
Long startTs, Long endTs, boolean useLatestOnly) {
StringBuilder sql = new StringBuilder();
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
if (useLatestOnly) {
sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ")
.append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n");
}
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
if (useLatestOnly) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n");
} else if (startTs != null) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID);
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new RatioCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,161 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.SessionCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Session command - Display session time information.
*
* Usage: session
*
* Shows timing information about the metrics session including first/last snapshots,
* duration, and snapshot cadence. Useful for understanding the data timeline.
*/
public class SessionCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "session";
}
@Override
public String getDescription() {
return "Display session time information and snapshot cadence";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
long startTime = System.currentTimeMillis();
String sql = """
SELECT
MIN(timestamp_ms) as first_snapshot_ms,
MAX(timestamp_ms) as last_snapshot_ms,
COUNT(DISTINCT timestamp_ms) as total_snapshots,
COUNT(*) as total_samples
FROM sample_value
""";
List<String> columns = new ArrayList<>();
columns.add("first_snapshot");
columns.add("last_snapshot");
columns.add("duration");
columns.add("total_snapshots");
columns.add("total_samples");
columns.add("avg_interval");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
long firstMs = rs.getLong("first_snapshot_ms");
long lastMs = rs.getLong("last_snapshot_ms");
int totalSnapshots = rs.getInt("total_snapshots");
int totalSamples = rs.getInt("total_samples");
long durationMs = lastMs - firstMs;
long avgIntervalMs = totalSnapshots > 1 ? durationMs / (totalSnapshots - 1) : 0;
Map<String, Object> row = new LinkedHashMap<>();
row.put("first_snapshot", new Timestamp(firstMs).toString());
row.put("last_snapshot", new Timestamp(lastMs).toString());
row.put("duration", formatDuration(durationMs));
row.put("total_snapshots", totalSnapshots);
row.put("total_samples", totalSamples);
row.put("avg_interval", formatDuration(avgIntervalMs));
rows.add(row);
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
// No parameters required for session command
}
@Override
public String getUsageExamples() {
return """
Examples:
# Show session information
session
# JSON format for scripting
session --format json
# Markdown for documentation
session --format markdown
""";
}
private String formatDuration(long millis) {
if (millis == 0) {
return "0s";
}
long seconds = millis / 1000;
long minutes = seconds / 60;
long hours = minutes / 60;
long days = hours / 24;
if (days > 0) {
return String.format("%dd %dh %dm", days, hours % 24, minutes % 60);
} else if (hours > 0) {
return String.format("%dh %dm %ds", hours, minutes % 60, seconds % 60);
} else if (minutes > 0) {
return String.format("%dm %ds", minutes, seconds % 60);
} else {
return String.format("%ds", seconds);
}
}
/**
* Standalone main method for direct execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new SessionCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,172 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.SqlCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* SQL command - Execute raw SQLite queries against the metrics database.
*
* Usage: sql --query "SELECT * FROM metric_family"
*
* Allows advanced users to write custom SQL queries for exploration,
* debugging, and analytics not covered by high-level commands.
*
* Safety:
* - Connection is read-only (no modifications possible)
* - Query timeout prevents runaway queries
* - Full SQLite dialect and functions available
*/
public class SqlCommand implements MetricsQueryCommand {
private static final int DEFAULT_QUERY_TIMEOUT_SECONDS = 30;
@Override
public String getName() {
return "sql";
}
@Override
public String getDescription() {
return "Execute raw SQLite queries (read-only)";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String sql = (String) params.get("query");
Integer timeoutSeconds = (Integer) params.getOrDefault("timeout", DEFAULT_QUERY_TIMEOUT_SECONDS);
long startTime = System.currentTimeMillis();
List<String> columns = new ArrayList<>();
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
ps.setQueryTimeout(timeoutSeconds);
try (ResultSet rs = ps.executeQuery()) {
// Extract column names from result set metadata
ResultSetMetaData metadata = rs.getMetaData();
int columnCount = metadata.getColumnCount();
for (int i = 1; i <= columnCount; i++) {
columns.add(metadata.getColumnName(i));
}
// Extract rows
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
for (int i = 1; i <= columnCount; i++) {
String columnName = metadata.getColumnName(i);
Object value = rs.getObject(i);
row.put(columnName, value);
}
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("query")) {
throw new InvalidQueryException("Missing required parameter: query");
}
Object query = params.get("query");
if (!(query instanceof String) || ((String) query).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'query' must be a non-empty SQL string");
}
String sql = ((String) query).trim().toUpperCase();
// Basic validation: ensure it's a SELECT query (read-only)
if (!sql.startsWith("SELECT") && !sql.startsWith("WITH") && !sql.startsWith("PRAGMA")) {
throw new InvalidQueryException(
"Only SELECT, WITH (CTE), and PRAGMA queries are allowed.\n" +
"Database is read-only to prevent modifications."
);
}
// Block dangerous pragmas
if (sql.contains("PRAGMA WRITABLE_SCHEMA") || sql.contains("PRAGMA CASE_SENSITIVE_LIKE")) {
throw new InvalidQueryException("This PRAGMA is not allowed");
}
// Validate timeout if present
if (params.containsKey("timeout")) {
Object timeout = params.get("timeout");
if (!(timeout instanceof Integer) || (Integer) timeout <= 0) {
throw new InvalidQueryException("Parameter 'timeout' must be a positive integer (seconds)");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# List all metric families
sql --query "SELECT name, type, unit FROM metric_family ORDER BY name"
# Get sample counts by metric
sql --query "SELECT sn.sample, COUNT(*) as count FROM sample_value sv JOIN sample_name sn ON sv.sample_name_id = sn.id GROUP BY sn.sample"
# Custom time-series query
sql --query "SELECT datetime(timestamp_ms/1000, 'unixepoch') as time, value FROM sample_value WHERE sample_name_id = 1 ORDER BY timestamp_ms"
# Schema inspection
sql --query "PRAGMA table_info(sample_value)"
# With timeout
sql --query "SELECT * FROM sample_value" --timeout 60
""";
}
/**
* Standalone main method for direct execution of SQL queries.
* Delegates to SqlCLI for picocli argument parsing.
*
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.SqlCommand --query "SELECT ..."
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new SqlCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,353 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.SummaryCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.*;
/**
* Summary command - Display comprehensive end-of-session metrics report.
*
* Usage: summary
*
* Provides an elaborate session summary focusing on NoSQLBench core metrics:
* - Session timeline and duration
* - Core operation metrics (cycles, result, tries, errors)
* - Activity breakdowns
* - Error analysis
* - Performance indicators
*
* Designed to give a complete picture of the session at a glance.
*/
public class SummaryCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "summary";
}
@Override
public String getDescription() {
return "Display comprehensive end-of-session metrics report";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
long startTime = System.currentTimeMillis();
// Build comprehensive summary with multiple sections
List<String> columns = new ArrayList<>();
columns.add("section");
columns.add("metric");
columns.add("value");
columns.add("details");
List<Map<String, Object>> rows = new ArrayList<>();
// Section 1: Session Overview
addSessionOverview(conn, rows);
// Section 2: Core Metrics Summary (cycles, result, tries, errors)
addCoreMetricsSummary(conn, rows);
// Section 3: Activity Breakdown
addActivityBreakdown(conn, rows);
// Section 4: Error Analysis
addErrorAnalysis(conn, rows);
// Section 5: All Available Metrics
addAllMetrics(conn, rows);
long executionTime = System.currentTimeMillis() - startTime;
// Build a single SQL representing all queries (for reference)
String combinedSql = "-- Multiple queries executed for comprehensive summary\n" +
"-- See SummaryCommand.java for details";
return new QueryResult(columns, rows, combinedSql, executionTime);
}
private void addSessionOverview(Connection conn, List<Map<String, Object>> rows) throws SQLException {
String sql = """
SELECT
MIN(timestamp_ms) as first_ts,
MAX(timestamp_ms) as last_ts,
COUNT(DISTINCT timestamp_ms) as snapshots,
COUNT(*) as total_samples
FROM sample_value
""";
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
long firstMs = rs.getLong("first_ts");
long lastMs = rs.getLong("last_ts");
int snapshots = rs.getInt("snapshots");
int totalSamples = rs.getInt("total_samples");
long durationMs = lastMs - firstMs;
rows.add(createRow("SESSION", "Time Range",
new Timestamp(firstMs) + " to " + new Timestamp(lastMs),
formatDuration(durationMs)));
rows.add(createRow("SESSION", "Snapshots", String.valueOf(snapshots),
"Total samples: " + totalSamples));
if (snapshots > 1) {
long avgInterval = durationMs / (snapshots - 1);
rows.add(createRow("SESSION", "Avg Interval", formatDuration(avgInterval),
"~" + (avgInterval / 1000) + "s cadence"));
}
}
}
}
private void addCoreMetricsSummary(Connection conn, List<Map<String, Object>> rows) throws SQLException {
// Query for core NoSQLBench metrics: cycles, result, tries, errors
String[] coreMetrics = {"cycles", "result", "tries", "errors"};
for (String metricBase : coreMetrics) {
String sql = """
SELECT
SUM(sv.value) as total_value,
COUNT(DISTINCT sv.label_set_id) as label_sets
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
WHERE sn.sample LIKE ?
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
""";
try (PreparedStatement ps = conn.prepareStatement(sql)) {
ps.setString(1, "%" + metricBase + "%");
try (ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
double total = rs.getDouble("total_value");
int labelSets = rs.getInt("label_sets");
if (labelSets > 0) {
rows.add(createRow("CORE METRICS", metricBase.toUpperCase(),
String.format("%.0f", total),
labelSets + " label set(s)"));
}
}
}
}
}
// Calculate error rate if we have both result and errors
calculateErrorRate(conn, rows);
}
private void calculateErrorRate(Connection conn, List<Map<String, Object>> rows) throws SQLException {
String sql = """
SELECT
SUM(CASE WHEN sn.sample LIKE '%result%' THEN sv.value ELSE 0 END) as results,
SUM(CASE WHEN sn.sample LIKE '%error%' THEN sv.value ELSE 0 END) as errors
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
WHERE sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
""";
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
double results = rs.getDouble("results");
double errors = rs.getDouble("errors");
if (results > 0) {
double errorRate = (errors / results) * 100.0;
rows.add(createRow("CORE METRICS", "Error Rate",
String.format("%.2f%%", errorRate),
String.format("%.0f errors / %.0f results", errors, results)));
}
}
}
}
private void addActivityBreakdown(Connection conn, List<Map<String, Object>> rows) throws SQLException {
// Group by activity label if it exists
String sql = """
SELECT
lv.value as activity,
SUM(sv.value) as total
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
JOIN label_key lk ON lsm.label_key_id = lk.id
JOIN label_value lv ON lsm.label_value_id = lv.id
WHERE lk.name = 'activity'
AND sn.sample LIKE '%ops%'
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
GROUP BY lv.value
ORDER BY total DESC
""";
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
String activity = rs.getString("activity");
double total = rs.getDouble("total");
rows.add(createRow("ACTIVITY", activity,
String.format("%.0f", total), "operations"));
}
}
}
private void addErrorAnalysis(Connection conn, List<Map<String, Object>> rows) throws SQLException {
String sql = """
SELECT
sn.sample,
SUM(sv.value) as total_errors
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
WHERE sn.sample LIKE '%error%'
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
GROUP BY sn.sample
HAVING total_errors > 0
ORDER BY total_errors DESC
""";
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
String sample = rs.getString("sample");
double errors = rs.getDouble("total_errors");
rows.add(createRow("ERRORS", sample,
String.format("%.0f", errors), "total errors"));
}
}
}
private void addAllMetrics(Connection conn, List<Map<String, Object>> rows) throws SQLException {
String sql = """
SELECT
mf.name AS metric_name,
mf.type AS metric_type,
COUNT(DISTINCT sv.label_set_id) AS unique_label_sets,
COUNT(*) AS total_samples,
MIN(sv.value) AS min_value,
MAX(sv.value) AS max_value
FROM metric_family mf
JOIN sample_name sn ON sn.metric_family_id = mf.id
JOIN sample_value sv ON sv.sample_name_id = sn.id
GROUP BY mf.id, mf.name, mf.type
ORDER BY mf.name
""";
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
String name = rs.getString("metric_name");
String type = rs.getString("metric_type");
int labelSets = rs.getInt("unique_label_sets");
int samples = rs.getInt("total_samples");
double min = rs.getDouble("min_value");
double max = rs.getDouble("max_value");
String details = String.format("type=%s, samples=%d, label_sets=%d, range=[%.1f, %.1f]",
type, samples, labelSets, min, max);
rows.add(createRow("ALL METRICS", name, "", details));
}
}
}
private Map<String, Object> createRow(String section, String metric, String value, String details) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("section", section);
row.put("metric", metric);
row.put("value", value);
row.put("details", details);
return row;
}
private String formatDuration(long millis) {
if (millis == 0) {
return "0s";
}
long seconds = millis / 1000;
long minutes = seconds / 60;
long hours = minutes / 60;
long days = hours / 24;
if (days > 0) {
return String.format("%dd %dh %dm", days, hours % 24, minutes % 60);
} else if (hours > 0) {
return String.format("%dh %dm %ds", hours, minutes % 60, seconds % 60);
} else if (minutes > 0) {
return String.format("%dm %ds", minutes, seconds % 60);
} else {
return String.format("%ds", seconds);
}
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
// No parameters required for summary command
}
@Override
public String getUsageExamples() {
return """
Examples:
# Show comprehensive session summary
summary
# JSON format for scripting
summary --format json
# Markdown for reports
summary --format markdown
# Use at end of session
nb5 run.yaml && nb5 mql summary
""";
}
/**
* Standalone main method for direct execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new SummaryCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,243 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.cli.TopKCLI;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.query.TimeWindowParser;
import io.nosqlbench.nb.mql.schema.MetricsSchema;
import picocli.CommandLine;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* TopK command - Find top N label sets by metric value.
*
* Usage: topk --metric <name> --n <count> [--window <duration>] [--labels key=val,...]
*
* Returns the N label sets with highest metric values.
* This is the PromQL topk() function.
*/
public class TopKCommand implements MetricsQueryCommand {
@Override
public String getName() {
return "topk";
}
@Override
public String getDescription() {
return "Find top N label sets by metric value";
}
@Override
public QueryResult execute(Connection conn, Map<String, Object> params)
throws SQLException, InvalidQueryException {
validate(params);
String metric = (String) params.get("metric");
Integer n = (Integer) params.get("n");
@SuppressWarnings("unchecked")
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
long startTime = System.currentTimeMillis();
// Determine time range (optional - defaults to latest)
Long startTimestampMs = null;
Long endTimestampMs = null;
boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last");
if (params.containsKey("window")) {
String window = (String) params.get("window");
long windowMs = TimeWindowParser.parseToMillis(window);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
} else if (params.containsKey("last")) {
String last = (String) params.get("last");
long windowMs = TimeWindowParser.parseToMillis(last);
endTimestampMs = getCurrentMaxTimestamp(conn);
startTimestampMs = endTimestampMs - windowMs;
}
String sql = buildQuery(labels, startTimestampMs, endTimestampMs, useLatestOnly, n);
List<String> columns = new ArrayList<>();
columns.add("value");
columns.add("labels");
List<Map<String, Object>> rows = new ArrayList<>();
try (PreparedStatement ps = conn.prepareStatement(sql)) {
int paramIndex = 1;
ps.setString(paramIndex++, metric);
if (!useLatestOnly && startTimestampMs != null) {
ps.setLong(paramIndex++, startTimestampMs);
ps.setLong(paramIndex++, endTimestampMs);
}
// Add label filter parameters
for (Map.Entry<String, String> label : labels.entrySet()) {
ps.setString(paramIndex++, label.getKey());
ps.setString(paramIndex++, label.getValue());
}
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("value", rs.getDouble("value"));
row.put("labels", rs.getString("labels"));
rows.add(row);
}
}
}
long executionTime = System.currentTimeMillis() - startTime;
return new QueryResult(columns, rows, sql, executionTime);
}
@Override
public void validate(Map<String, Object> params) throws InvalidQueryException {
if (!params.containsKey("metric")) {
throw new InvalidQueryException("Missing required parameter: metric");
}
if (!params.containsKey("n")) {
throw new InvalidQueryException("Missing required parameter: n (number of top results)");
}
Object metric = params.get("metric");
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
}
Object n = params.get("n");
if (!(n instanceof Integer) || (Integer) n <= 0) {
throw new InvalidQueryException("Parameter 'n' must be a positive integer");
}
// Validate labels if present
if (params.containsKey("labels")) {
Object labels = params.get("labels");
if (!(labels instanceof Map)) {
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
}
}
}
@Override
public String getUsageExamples() {
return """
Examples:
# Top 5 endpoints by request count
topk --metric api_requests_total --n 5
# Top 3 slowest operations
topk --metric operation_latency --n 3
# Top 10 in last hour
topk --metric requests_total --n 10 --window 1h
# Top 5 errors
topk --metric errors_total --n 5 --labels status=500
""";
}
private String buildQuery(Map<String, String> labelFilters, Long startTs, Long endTs,
boolean useLatestOnly, int limit) {
StringBuilder sql = new StringBuilder();
sql.append("WITH latest_values AS (\n");
sql.append(" SELECT\n");
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
if (useLatestOnly) {
sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ")
.append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n");
}
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
if (useLatestOnly) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n");
} else if (startTs != null) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
}
// Add label filters
for (String labelKey : labelFilters.keySet()) {
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
sql.append(" )\n");
}
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
sql.append(")\n");
sql.append("SELECT value, labels\n");
sql.append("FROM latest_values\n");
sql.append("ORDER BY value DESC\n");
sql.append("LIMIT ").append(limit);
return sql.toString();
}
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
MetricsSchema.TABLE_SAMPLE_VALUE;
try (PreparedStatement ps = conn.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
}
throw new SQLException("No data in database");
}
}
/**
* Standalone main method for direct execution.
*/
public static void main(String[] args) {
int exitCode = new CommandLine(new TopKCLI()).execute(args);
System.exit(exitCode);
}
}
@@ -0,0 +1,78 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import io.nosqlbench.nb.mql.query.QueryResult;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Formats query results as CSV for spreadsheet import.
*/
public class CsvFormatter implements ResultFormatter {
private final boolean includeHeaders;
public CsvFormatter() {
this(true);
}
public CsvFormatter(boolean includeHeaders) {
this.includeHeaders = includeHeaders;
}
@Override
public String format(QueryResult result) {
StringWriter writer = new StringWriter();
try {
CSVFormat format = includeHeaders
? CSVFormat.DEFAULT.builder().setHeader(result.columns().toArray(new String[0])).build()
: CSVFormat.DEFAULT;
try (CSVPrinter printer = new CSVPrinter(writer, format)) {
for (Map<String, Object> row : result.rows()) {
List<Object> values = new ArrayList<>();
for (String column : result.columns()) {
values.add(row.get(column));
}
printer.printRecord(values);
}
}
} catch (IOException e) {
throw new RuntimeException("Failed to format CSV", e);
}
return writer.toString();
}
@Override
public String getName() {
return "csv";
}
@Override
public String getFileExtension() {
return "csv";
}
}
@@ -0,0 +1,67 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.mql.query.QueryResult;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* Formats query results as JSON for programmatic consumption.
*/
public class JsonFormatter implements ResultFormatter {
private final Gson gson;
private final boolean pretty;
public JsonFormatter() {
this(false);
}
public JsonFormatter(boolean pretty) {
this.pretty = pretty;
GsonBuilder builder = new GsonBuilder();
if (pretty) {
builder.setPrettyPrinting();
}
this.gson = builder.create();
}
@Override
public String format(QueryResult result) {
Map<String, Object> output = new LinkedHashMap<>();
output.put("columns", result.columns());
output.put("rows", result.rows());
output.put("rowCount", result.rowCount());
output.put("executionTimeMs", result.executionTimeMs());
return gson.toJson(output);
}
@Override
public String getName() {
return "json";
}
@Override
public String getFileExtension() {
return "json";
}
}
@@ -0,0 +1,60 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import com.google.gson.Gson;
import io.nosqlbench.nb.mql.query.QueryResult;
import java.util.Map;
/**
* Formats query results as JSONL (JSON Lines) for streaming data processing.
* Each row is output as a single JSON object on its own line, with no array wrapper.
* This format is efficient for large result sets and compatible with streaming processors.
*/
public class JsonLinesFormatter implements ResultFormatter {
private final Gson gson;
public JsonLinesFormatter() {
// Compact JSON (no pretty printing for JSONL)
this.gson = new Gson();
}
@Override
public String format(QueryResult result) {
StringBuilder sb = new StringBuilder();
// Each row is a separate JSON object on its own line
for (Map<String, Object> row : result.rows()) {
sb.append(gson.toJson(row));
sb.append("\n");
}
return sb.toString();
}
@Override
public String getName() {
return "jsonl";
}
@Override
public String getFileExtension() {
return "jsonl";
}
}
@@ -0,0 +1,144 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import io.nosqlbench.nb.mql.query.QueryResult;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Formats query results as GitHub-flavored Markdown tables.
* Output can be directly copy-pasted into markdown documentation.
*/
public class MarkdownFormatter implements ResultFormatter {
private static final int DEFAULT_MAX_COLUMN_WIDTH = 50;
@Override
public String format(QueryResult result) {
if (result.isEmpty()) {
return "*No results found.*\n";
}
List<String> columns = result.columns();
List<Map<String, Object>> rows = result.rows();
// Calculate column widths
Map<String, Integer> columnWidths = calculateColumnWidths(columns, rows);
StringBuilder sb = new StringBuilder();
// Header
sb.append("| ");
for (int i = 0; i < columns.size(); i++) {
String column = columns.get(i);
sb.append(padRight(column, columnWidths.get(column)));
if (i < columns.size() - 1) {
sb.append(" | ");
}
}
sb.append(" |\n");
// Separator row
sb.append("|");
for (int i = 0; i < columns.size(); i++) {
String column = columns.get(i);
sb.append("-".repeat(columnWidths.get(column) + 2)); // +2 for padding spaces
if (i < columns.size() - 1) {
sb.append("|");
}
}
sb.append("|\n");
// Data rows
for (Map<String, Object> row : rows) {
sb.append("| ");
for (int i = 0; i < columns.size(); i++) {
String column = columns.get(i);
Object value = row.get(column);
String strValue = value != null ? escapeMarkdown(value.toString()) : "";
// Truncate if too long
if (strValue.length() > DEFAULT_MAX_COLUMN_WIDTH) {
strValue = strValue.substring(0, DEFAULT_MAX_COLUMN_WIDTH - 3) + "...";
}
sb.append(padRight(strValue, columnWidths.get(column)));
if (i < columns.size() - 1) {
sb.append(" | ");
}
}
sb.append(" |\n");
}
// Footer with metadata
sb.append("\n");
sb.append("*").append(result.rowCount()).append(" row");
if (result.rowCount() != 1) {
sb.append("s");
}
sb.append(" (").append(result.executionTimeMs()).append("ms)*\n");
return sb.toString();
}
@Override
public String getName() {
return "markdown";
}
@Override
public String getFileExtension() {
return "md";
}
private Map<String, Integer> calculateColumnWidths(List<String> columns, List<Map<String, Object>> rows) {
Map<String, Integer> widths = new java.util.LinkedHashMap<>();
// Initialize with header widths
for (String column : columns) {
widths.put(column, column.length());
}
// Check data widths
for (Map<String, Object> row : rows) {
for (String column : columns) {
Object value = row.get(column);
String strValue = value != null ? value.toString() : "";
int currentWidth = widths.get(column);
int valueWidth = Math.min(strValue.length(), DEFAULT_MAX_COLUMN_WIDTH);
widths.put(column, Math.max(currentWidth, valueWidth));
}
}
return widths;
}
private String padRight(String str, int width) {
if (str.length() >= width) {
return str;
}
return str + " ".repeat(width - str.length());
}
private String escapeMarkdown(String str) {
// Escape pipe characters which would break the table
return str.replace("|", "\\|");
}
}
@@ -0,0 +1,162 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
/**
* Enum representing all supported output formats for query results.
* Provides type safety and validation for format selection in CLI.
*/
public enum OutputFormat {
/**
* ASCII tables for console output (default).
*/
TABLE("table", "txt", "ASCII tables for console output") {
@Override
public ResultFormatter createFormatter() {
return new TableFormatter();
}
},
/**
* Comma-separated values for spreadsheet import.
*/
CSV("csv", "csv", "Comma-separated values for Excel/Sheets") {
@Override
public ResultFormatter createFormatter() {
return new CsvFormatter();
}
},
/**
* Tab-separated values for Unix text processing.
*/
TSV("tsv", "tsv", "Tab-separated values for awk/cut/paste") {
@Override
public ResultFormatter createFormatter() {
return new TsvFormatter();
}
},
/**
* Pretty-printed JSON for programmatic use.
*/
JSON("json", "json", "Pretty-printed JSON with metadata") {
@Override
public ResultFormatter createFormatter() {
return new JsonFormatter(true);
}
},
/**
* JSON Lines (one JSON object per line) for streaming.
*/
JSONL("jsonl", "jsonl", "JSON Lines for streaming processors") {
@Override
public ResultFormatter createFormatter() {
return new JsonLinesFormatter();
}
},
/**
* GitHub-flavored Markdown tables for documentation.
*/
MARKDOWN("markdown", "md", "Markdown tables for documentation") {
@Override
public ResultFormatter createFormatter() {
return new MarkdownFormatter();
}
};
private final String name;
private final String extension;
private final String description;
OutputFormat(String name, String extension, String description) {
this.name = name;
this.extension = extension;
this.description = description;
}
/**
* Create a formatter instance for this format.
* Each enum value knows how to create its own formatter.
*
* @return ResultFormatter instance
*/
public abstract ResultFormatter createFormatter();
/**
* Get the format name as it appears in CLI arguments.
*/
public String getName() {
return name;
}
/**
* Get the file extension for this format.
*/
public String getExtension() {
return extension;
}
/**
* Get a human-readable description.
*/
public String getDescription() {
return description;
}
/**
* Parse a format name string to an enum value.
*
* @param name Format name (case-insensitive)
* @return The OutputFormat enum, or null if not found
*/
public static OutputFormat fromString(String name) {
if (name == null || name.trim().isEmpty()) {
return TABLE; // Default
}
String normalized = name.toLowerCase().trim();
for (OutputFormat format : values()) {
if (format.name.equals(normalized)) {
return format;
}
}
return null; // Not found
}
/**
* Get a comma-separated list of all format names.
* Useful for CLI help text.
*/
public static String getAllFormatNames() {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < values().length; i++) {
sb.append(values()[i].name);
if (i < values().length - 1) {
sb.append(", ");
}
}
return sb.toString();
}
@Override
public String toString() {
return name;
}
}
@@ -0,0 +1,47 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import io.nosqlbench.nb.mql.query.QueryResult;
/**
* Interface for formatting query results in different output formats.
*/
public interface ResultFormatter {
/**
* Format a query result as a string.
*
* @param result The query result to format
* @return Formatted string representation
*/
String format(QueryResult result);
/**
* Get the name of this formatter (e.g., "table", "json", "csv").
*
* @return Formatter name
*/
String getName();
/**
* Get the file extension for this format (e.g., "txt", "json", "csv").
*
* @return File extension without the dot
*/
String getFileExtension();
}
@@ -0,0 +1,221 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import io.nosqlbench.nb.mql.query.QueryResult;
import java.util.*;
/**
* Custom markdown formatter for summary command output.
* Creates a nicely structured report with tree views and sections.
*/
public class SummaryMarkdownFormatter implements ResultFormatter {
@Override
public String format(QueryResult result) {
if (result.isEmpty()) {
return "*No data available.*\n";
}
StringBuilder md = new StringBuilder();
// Title
md.append("# NoSQLBench Session Summary\n\n");
// Group rows by section
Map<String, List<Map<String, Object>>> sections = new LinkedHashMap<>();
for (Map<String, Object> row : result.rows()) {
String section = (String) row.get("section");
sections.computeIfAbsent(section, k -> new ArrayList<>()).add(row);
}
// Render each section
for (Map.Entry<String, List<Map<String, Object>>> entry : sections.entrySet()) {
String sectionName = entry.getKey();
List<Map<String, Object>> sectionRows = entry.getValue();
if ("SESSION".equals(sectionName)) {
renderSessionSection(md, sectionRows);
} else if ("CORE METRICS".equals(sectionName)) {
renderCoreMetricsSection(md, sectionRows);
} else if ("ACTIVITY".equals(sectionName)) {
renderActivitySection(md, sectionRows);
} else if ("ERRORS".equals(sectionName)) {
renderErrorsSection(md, sectionRows);
} else if ("ALL METRICS".equals(sectionName)) {
renderAllMetricsTree(md, sectionRows);
} else {
renderGenericSection(md, sectionName, sectionRows);
}
md.append("\n");
}
// Footer
md.append("---\n\n");
md.append(String.format("*Report generated in %dms*\n", result.executionTimeMs()));
return md.toString();
}
private void renderSessionSection(StringBuilder md, List<Map<String, Object>> rows) {
md.append("## 📊 Session Overview\n\n");
for (Map<String, Object> row : rows) {
String metric = (String) row.get("metric");
String value = (String) row.get("value");
String details = (String) row.get("details");
md.append("**").append(metric).append(":** \n");
md.append("`").append(value).append("`");
if (details != null && !details.isEmpty()) {
md.append(" — *").append(details).append("*");
}
md.append("\n\n");
}
}
private void renderCoreMetricsSection(StringBuilder md, List<Map<String, Object>> rows) {
md.append("## 🎯 Core Metrics\n\n");
md.append("| Metric | Value | Details |\n");
md.append("|--------|-------|----------|\n");
for (Map<String, Object> row : rows) {
String metric = (String) row.get("metric");
String value = (String) row.get("value");
String details = (String) row.get("details");
md.append("| **").append(metric).append("** | ");
md.append("`").append(value).append("` | ");
md.append(details != null ? details : "");
md.append(" |\n");
}
md.append("\n");
}
private void renderActivitySection(StringBuilder md, List<Map<String, Object>> rows) {
md.append("## 📈 Activity Breakdown\n\n");
md.append("```\n");
long total = rows.stream()
.mapToLong(row -> {
String value = (String) row.get("value");
return (long) Double.parseDouble(value);
})
.sum();
for (Map<String, Object> row : rows) {
String activity = (String) row.get("metric");
String value = (String) row.get("value");
long count = (long) Double.parseDouble(value);
double percentage = (count * 100.0) / total;
md.append(String.format("%-15s %,12d ops (%5.1f%%)\n",
activity + ":", count, percentage));
}
md.append(String.format("%-15s %,12d ops\n", "TOTAL:", total));
md.append("```\n\n");
}
private void renderErrorsSection(StringBuilder md, List<Map<String, Object>> rows) {
md.append("## ⚠️ Errors\n\n");
if (rows.isEmpty()) {
md.append("*No errors recorded.*\n\n");
return;
}
md.append("| Error Type | Count |\n");
md.append("|------------|-------|\n");
for (Map<String, Object> row : rows) {
String metric = (String) row.get("metric");
String value = (String) row.get("value");
md.append("| `").append(metric).append("` | ");
md.append("**").append(value).append("** |\n");
}
md.append("\n");
}
private void renderAllMetricsTree(StringBuilder md, List<Map<String, Object>> rows) {
md.append("## 📋 Metrics Inventory\n\n");
md.append("```\n");
md.append("Metrics Database\n");
for (int i = 0; i < rows.size(); i++) {
Map<String, Object> row = rows.get(i);
String name = (String) row.get("metric");
String details = (String) row.get("details");
boolean isLast = (i == rows.size() - 1);
String prefix = isLast ? "└── " : "├── ";
md.append(prefix).append(name).append("\n");
if (details != null && !details.isEmpty()) {
String detailPrefix = isLast ? " " : "│ ";
// Parse details and format as tree branches
// Handle range=[min, max] specially to keep it on one line
String[] parts = details.split(", (?![^\\[]*\\])");
for (int j = 0; j < parts.length; j++) {
boolean isLastDetail = (j == parts.length - 1);
String branchPrefix = isLastDetail ? "└── " : "├── ";
md.append(detailPrefix).append(branchPrefix).append(parts[j]).append("\n");
}
}
}
md.append("```\n\n");
}
private void renderGenericSection(StringBuilder md, String sectionName, List<Map<String, Object>> rows) {
md.append("## ").append(sectionName).append("\n\n");
md.append("| Metric | Value | Details |\n");
md.append("|--------|-------|----------|\n");
for (Map<String, Object> row : rows) {
String metric = (String) row.get("metric");
String value = (String) row.get("value");
String details = (String) row.get("details");
md.append("| ").append(metric).append(" | ");
md.append(value != null ? value : "").append(" | ");
md.append(details != null ? details : "");
md.append(" |\n");
}
md.append("\n");
}
@Override
public String getName() {
return "summary-markdown";
}
@Override
public String getFileExtension() {
return "md";
}
}
@@ -0,0 +1,152 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import io.nosqlbench.nb.mql.query.QueryResult;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Formats query results as ASCII tables suitable for console output.
*/
public class TableFormatter implements ResultFormatter {
private static final int DEFAULT_MAX_COLUMN_WIDTH = 50;
private static final String COLUMN_SEPARATOR = " | ";
private static final String HEADER_SEPARATOR_CHAR = "-";
@Override
public String format(QueryResult result) {
if (result.isEmpty()) {
return "No results found.\n";
}
List<String> columns = result.columns();
List<Map<String, Object>> rows = result.rows();
// Calculate column widths
Map<String, Integer> columnWidths = calculateColumnWidths(columns, rows);
StringBuilder sb = new StringBuilder();
// Header
sb.append(formatRow(columns, columnWidths, columns));
sb.append("\n");
// Separator
sb.append(formatSeparator(columns, columnWidths));
sb.append("\n");
// Data rows
for (Map<String, Object> row : rows) {
List<String> values = new ArrayList<>();
for (String column : columns) {
Object value = row.get(column);
values.add(value != null ? value.toString() : "null");
}
sb.append(formatRow(values, columnWidths, columns));
sb.append("\n");
}
// Footer with row count
sb.append("\n");
sb.append(result.rowCount()).append(" row");
if (result.rowCount() != 1) {
sb.append("s");
}
sb.append(" (").append(result.executionTimeMs()).append("ms)\n");
return sb.toString();
}
@Override
public String getName() {
return "table";
}
@Override
public String getFileExtension() {
return "txt";
}
private Map<String, Integer> calculateColumnWidths(List<String> columns, List<Map<String, Object>> rows) {
Map<String, Integer> widths = new java.util.LinkedHashMap<>();
// Initialize with header widths
for (String column : columns) {
widths.put(column, column.length());
}
// Check data widths
for (Map<String, Object> row : rows) {
for (String column : columns) {
Object value = row.get(column);
String strValue = value != null ? value.toString() : "null";
int currentWidth = widths.get(column);
int valueWidth = Math.min(strValue.length(), DEFAULT_MAX_COLUMN_WIDTH);
widths.put(column, Math.max(currentWidth, valueWidth));
}
}
return widths;
}
private String formatRow(List<String> values, Map<String, Integer> columnWidths, List<String> columns) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < columns.size(); i++) {
String column = columns.get(i);
String value = values.get(i);
int width = columnWidths.get(column);
// Truncate if too long
if (value.length() > DEFAULT_MAX_COLUMN_WIDTH) {
value = value.substring(0, DEFAULT_MAX_COLUMN_WIDTH - 3) + "...";
}
// Pad to width
sb.append(padRight(value, width));
if (i < columns.size() - 1) {
sb.append(COLUMN_SEPARATOR);
}
}
return sb.toString();
}
private String formatSeparator(List<String> columns, Map<String, Integer> columnWidths) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < columns.size(); i++) {
String column = columns.get(i);
int width = columnWidths.get(column);
sb.append(HEADER_SEPARATOR_CHAR.repeat(width));
if (i < columns.size() - 1) {
sb.append("-+-");
}
}
return sb.toString();
}
private String padRight(String str, int width) {
if (str.length() >= width) {
return str;
}
return str + " ".repeat(width - str.length());
}
}
@@ -0,0 +1,79 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.format;
import io.nosqlbench.nb.mql.query.QueryResult;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Formats query results as TSV (Tab-Separated Values) for data processing tools.
* Compatible with awk, cut, paste, and other Unix text processing utilities.
*/
public class TsvFormatter implements ResultFormatter {
private final boolean includeHeaders;
public TsvFormatter() {
this(true);
}
public TsvFormatter(boolean includeHeaders) {
this.includeHeaders = includeHeaders;
}
@Override
public String format(QueryResult result) {
StringBuilder sb = new StringBuilder();
// Header row
if (includeHeaders) {
sb.append(String.join("\t", result.columns()));
sb.append("\n");
}
// Data rows
for (Map<String, Object> row : result.rows()) {
List<String> values = new ArrayList<>();
for (String column : result.columns()) {
Object value = row.get(column);
// TSV doesn't require quoting - just convert to string
// Tabs and newlines in data should be replaced
String strValue = value != null ? value.toString() : "";
strValue = strValue.replace("\t", " ").replace("\n", " ").replace("\r", "");
values.add(strValue);
}
sb.append(String.join("\t", values));
sb.append("\n");
}
return sb.toString();
}
@Override
public String getName() {
return "tsv";
}
@Override
public String getFileExtension() {
return "tsv";
}
}
@@ -0,0 +1,32 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.query;
/**
* Exception thrown when query parameters are invalid.
* Should provide clear, actionable error messages to help users correct their queries.
*/
public class InvalidQueryException extends Exception {
public InvalidQueryException(String message) {
super(message);
}
public InvalidQueryException(String message, Throwable cause) {
super(message, cause);
}
}
@@ -0,0 +1,67 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.query;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Map;
/**
* Interface for all MetricsQL query commands.
* Each command translates high-level query parameters into SQL and executes against
* the NoSQLBench metrics database.
*/
public interface MetricsQueryCommand {
/**
* Get the command name (e.g., "rate", "instant", "agg").
*/
String getName();
/**
* Get a human-readable description of what this command does.
*/
String getDescription();
/**
* Execute the query command.
*
* @param conn Database connection
* @param params Query parameters (metric name, time windows, labels, etc.)
* @return Query result with columns and rows
* @throws SQLException If query execution fails
* @throws InvalidQueryException If parameters are invalid
*/
QueryResult execute(Connection conn, Map<String, Object> params) throws SQLException, InvalidQueryException;
/**
* Validate query parameters before execution.
* Should throw InvalidQueryException with clear error messages if parameters are invalid.
*
* @param params Query parameters to validate
* @throws InvalidQueryException If parameters are invalid
*/
void validate(Map<String, Object> params) throws InvalidQueryException;
/**
* Get usage examples for this command.
* Used for help text and documentation.
*/
default String getUsageExamples() {
return "No examples available.";
}
}
@@ -0,0 +1,49 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.query;
import java.util.List;
import java.util.Map;
/**
* Represents the result of a metrics query.
*
* @param columns List of column names in order
* @param rows List of row data as maps (column name -> value)
* @param executedSQL The SQL query that was executed (for debugging/explain)
* @param executionTimeMs Time taken to execute the query in milliseconds
*/
public record QueryResult(
List<String> columns,
List<Map<String, Object>> rows,
String executedSQL,
long executionTimeMs
) {
/**
* Get the number of rows returned.
*/
public int rowCount() {
return rows != null ? rows.size() : 0;
}
/**
* Check if the result is empty.
*/
public boolean isEmpty() {
return rowCount() == 0;
}
}
@@ -0,0 +1,160 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.query;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Parses time window specifications like "5m", "1h", "30s" into milliseconds.
* Supports PromQL/MetricsQL time duration format.
*/
public class TimeWindowParser {
// Pattern: number followed by time unit (s, m, h, d, w, y)
private static final Pattern TIME_WINDOW_PATTERN = Pattern.compile("^(\\d+(?:\\.\\d+)?)([smhdwy])$");
/**
* Parse a time window string into milliseconds.
*
* Supported formats:
* - "5s" = 5 seconds = 5000ms
* - "10m" = 10 minutes = 600000ms
* - "2h" = 2 hours = 7200000ms
* - "1d" = 1 day = 86400000ms
* - "1w" = 1 week = 604800000ms
* - "1y" = 1 year (365 days) = 31536000000ms
*
* @param window Time window string (e.g., "5m", "1h")
* @return Time window in milliseconds
* @throws InvalidQueryException If the format is invalid
*/
public static long parseToMillis(String window) throws InvalidQueryException {
if (window == null || window.trim().isEmpty()) {
throw new InvalidQueryException("Time window cannot be null or empty");
}
String normalized = window.trim().toLowerCase();
Matcher matcher = TIME_WINDOW_PATTERN.matcher(normalized);
if (!matcher.matches()) {
throw new InvalidQueryException(
"Invalid time window format: '" + window + "'\n" +
"Expected format: <number><unit> where unit is one of: s, m, h, d, w, y\n" +
"Examples: 5s, 10m, 2h, 1d"
);
}
double value;
try {
value = Double.parseDouble(matcher.group(1));
} catch (NumberFormatException e) {
throw new InvalidQueryException("Invalid numeric value in time window: " + window, e);
}
if (value <= 0) {
throw new InvalidQueryException("Time window must be positive, got: " + value);
}
String unit = matcher.group(2);
long millisPerUnit = getMillisPerUnit(unit);
// Check for overflow
double resultMs = value * millisPerUnit;
if (resultMs > Long.MAX_VALUE) {
throw new InvalidQueryException("Time window too large: " + window);
}
return (long) resultMs;
}
/**
* Parse a time window string, throwing RuntimeException on error.
* Useful for constants where you know the format is valid.
*
* @param window Time window string
* @return Time window in milliseconds
* @throws RuntimeException If parsing fails
*/
public static long parseToMillisUnchecked(String window) {
try {
return parseToMillis(window);
} catch (InvalidQueryException e) {
throw new RuntimeException("Failed to parse time window: " + window, e);
}
}
/**
* Format milliseconds back to a human-readable time window string.
* Chooses the most appropriate unit automatically.
*
* @param millis Time in milliseconds
* @return Human-readable string (e.g., "5m", "2h")
*/
public static String formatMillis(long millis) {
if (millis == 0) {
return "0s";
}
// Try to find the largest unit that divides evenly
if (millis % MILLIS_PER_YEAR == 0) {
return (millis / MILLIS_PER_YEAR) + "y";
}
if (millis % MILLIS_PER_WEEK == 0) {
return (millis / MILLIS_PER_WEEK) + "w";
}
if (millis % MILLIS_PER_DAY == 0) {
return (millis / MILLIS_PER_DAY) + "d";
}
if (millis % MILLIS_PER_HOUR == 0) {
return (millis / MILLIS_PER_HOUR) + "h";
}
if (millis % MILLIS_PER_MINUTE == 0) {
return (millis / MILLIS_PER_MINUTE) + "m";
}
if (millis % MILLIS_PER_SECOND == 0) {
return (millis / MILLIS_PER_SECOND) + "s";
}
// Fallback to milliseconds
return millis + "ms";
}
// Time unit constants
private static final long MILLIS_PER_SECOND = 1000L;
private static final long MILLIS_PER_MINUTE = 60L * MILLIS_PER_SECOND;
private static final long MILLIS_PER_HOUR = 60L * MILLIS_PER_MINUTE;
private static final long MILLIS_PER_DAY = 24L * MILLIS_PER_HOUR;
private static final long MILLIS_PER_WEEK = 7L * MILLIS_PER_DAY;
private static final long MILLIS_PER_YEAR = 365L * MILLIS_PER_DAY; // No leap years in PromQL
private static long getMillisPerUnit(String unit) throws InvalidQueryException {
return switch (unit) {
case "s" -> MILLIS_PER_SECOND;
case "m" -> MILLIS_PER_MINUTE;
case "h" -> MILLIS_PER_HOUR;
case "d" -> MILLIS_PER_DAY;
case "w" -> MILLIS_PER_WEEK;
case "y" -> MILLIS_PER_YEAR;
default -> throw new InvalidQueryException("Unknown time unit: " + unit);
};
}
private TimeWindowParser() {
// Utility class - prevent instantiation
}
}
@@ -0,0 +1,50 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.schema;
import java.util.Map;
import java.util.StringJoiner;
/**
* Utility for computing label set hashes compatible with SqliteSnapshotReporter.
* This logic is intentionally duplicated from SqliteSnapshotReporter to maintain
* module independence (nb-api doesn't depend on nb-mql-api).
*/
public class LabelSetResolver {
/**
* Compute a deterministic hash string for a label set.
* Format: {key1=value1,key2=value2,...} with keys sorted alphabetically.
*
* @param labels Map of label key-value pairs
* @return Hash string representation
*/
public static String computeHash(Map<String, String> labels) {
if (labels == null || labels.isEmpty()) {
return "{}";
}
StringJoiner joiner = new StringJoiner(",", "{", "}");
labels.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.forEach(entry -> joiner.add(entry.getKey() + "=" + entry.getValue()));
return joiner.toString();
}
private LabelSetResolver() {
// Utility class - prevent instantiation
}
}
@@ -0,0 +1,118 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.schema;
import org.sqlite.SQLiteConfig;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
/**
* Manages read-only connections to NoSQLBench SQLite metrics databases.
* Supports querying both live sessions (via logs/metrics.db symlink) and
* historical session files.
*/
public class MetricsDatabaseReader {
private static final Path DEFAULT_DB_PATH = Path.of("logs/metrics.db");
private static final int DEFAULT_BUSY_TIMEOUT_MS = 5000;
static {
// Explicitly load SQLite JDBC driver for standalone JAR usage
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new RuntimeException("SQLite JDBC driver not found", e);
}
}
/**
* Connect to the default metrics database (logs/metrics.db).
* This typically points to the current session via symlink.
*
* @return Read-only database connection
* @throws SQLException If connection fails or database doesn't exist
*/
public static Connection connect() throws SQLException {
return connect(DEFAULT_DB_PATH);
}
/**
* Connect to a specific metrics database file.
*
* @param dbPath Path to the database file (can be a symlink)
* @return Read-only database connection
* @throws SQLException If connection fails or database doesn't exist
*/
public static Connection connect(Path dbPath) throws SQLException {
return connect(dbPath, DEFAULT_BUSY_TIMEOUT_MS);
}
/**
* Connect to a specific metrics database with custom busy timeout.
*
* @param dbPath Path to the database file
* @param busyTimeoutMs Timeout in milliseconds for handling database locks
* @return Read-only database connection
* @throws SQLException If connection fails or database doesn't exist
*/
public static Connection connect(Path dbPath, int busyTimeoutMs) throws SQLException {
Path resolvedPath = (dbPath != null) ? dbPath : DEFAULT_DB_PATH;
// Check if database file exists
if (!Files.exists(resolvedPath)) {
throw new SQLException(
"Database not found at: " + resolvedPath.toAbsolutePath() + "\n" +
"Hint: Ensure a NoSQLBench session is running or specify a session file with --db <path>"
);
}
// Build JDBC URL
String jdbcUrl = "jdbc:sqlite:" + resolvedPath.toAbsolutePath();
// Configure SQLite for read-only access
SQLiteConfig config = new SQLiteConfig();
config.setReadOnly(true);
config.setBusyTimeout(busyTimeoutMs);
// Create connection with config
return DriverManager.getConnection(jdbcUrl, config.toProperties());
}
/**
* Connect to a database specified by JDBC URL string.
* Useful for testing with in-memory databases.
*
* @param jdbcUrl JDBC URL (e.g., "jdbc:sqlite::memory:")
* @return Read-only database connection
* @throws SQLException If connection fails
*/
public static Connection connectByUrl(String jdbcUrl) throws SQLException {
SQLiteConfig config = new SQLiteConfig();
config.setReadOnly(true);
config.setBusyTimeout(DEFAULT_BUSY_TIMEOUT_MS);
return DriverManager.getConnection(jdbcUrl, config.toProperties());
}
private MetricsDatabaseReader() {
// Utility class - prevent instantiation
}
}
@@ -0,0 +1,169 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.schema;
/**
* Schema constants for NoSQLBench SQLite metrics database.
* This schema is defined and created by SqliteSnapshotReporter in nb-api.
* These constants provide a read-only view for query operations.
*/
public class MetricsSchema {
// Table names
public static final String TABLE_METRIC_FAMILY = "metric_family";
public static final String TABLE_SAMPLE_NAME = "sample_name";
public static final String TABLE_SAMPLE_VALUE = "sample_value";
public static final String TABLE_SAMPLE_QUANTILE = "sample_quantile";
public static final String TABLE_SAMPLE_RATE = "sample_rate";
public static final String TABLE_SAMPLE_HISTOGRAM = "sample_histogram";
public static final String TABLE_LABEL_KEY = "label_key";
public static final String TABLE_LABEL_VALUE = "label_value";
public static final String TABLE_LABEL_SET = "label_set";
public static final String TABLE_LABEL_SET_MEMBERSHIP = "label_set_membership";
// metric_family columns
public static final String COL_MF_ID = "id";
public static final String COL_MF_NAME = "name";
public static final String COL_MF_HELP = "help";
public static final String COL_MF_UNIT = "unit";
public static final String COL_MF_TYPE = "type";
// sample_name columns
public static final String COL_SN_ID = "id";
public static final String COL_SN_METRIC_FAMILY_ID = "metric_family_id";
public static final String COL_SN_SAMPLE = "sample";
// sample_value columns
public static final String COL_SV_ID = "id";
public static final String COL_SV_SAMPLE_NAME_ID = "sample_name_id";
public static final String COL_SV_LABEL_SET_ID = "label_set_id";
public static final String COL_SV_TIMESTAMP_MS = "timestamp_ms";
public static final String COL_SV_VALUE = "value";
public static final String COL_SV_COUNT = "count";
public static final String COL_SV_SUM = "sum";
public static final String COL_SV_MIN = "min";
public static final String COL_SV_MAX = "max";
public static final String COL_SV_MEAN = "mean";
public static final String COL_SV_STDDEV = "stddev";
// sample_quantile columns
public static final String COL_SQ_SAMPLE_VALUE_ID = "sample_value_id";
public static final String COL_SQ_QUANTILE = "quantile";
public static final String COL_SQ_QUANTILE_VALUE = "quantile_value";
// sample_rate columns
public static final String COL_SR_SAMPLE_VALUE_ID = "sample_value_id";
public static final String COL_SR_RATE_TYPE = "rate_type";
public static final String COL_SR_RATE_VALUE = "rate_value";
// sample_histogram columns
public static final String COL_SH_SAMPLE_VALUE_ID = "sample_value_id";
public static final String COL_SH_START_SECONDS = "start_seconds";
public static final String COL_SH_INTERVAL_SECONDS = "interval_seconds";
public static final String COL_SH_MAX_VALUE = "max_value";
public static final String COL_SH_HISTOGRAM_BASE64 = "histogram_base64";
// label_key columns
public static final String COL_LK_ID = "id";
public static final String COL_LK_NAME = "name";
// label_value columns
public static final String COL_LV_ID = "id";
public static final String COL_LV_VALUE = "value";
// label_set columns
public static final String COL_LS_ID = "id";
public static final String COL_LS_HASH = "hash";
// label_set_membership columns
public static final String COL_LSM_LABEL_SET_ID = "label_set_id";
public static final String COL_LSM_LABEL_KEY_ID = "label_key_id";
public static final String COL_LSM_LABEL_VALUE_ID = "label_value_id";
// Common SQL fragments for building queries
/**
* SQL fragment to join sample_value with sample_name.
* Usage: FROM sample_value sv JOIN sample_name sn ON ...
*/
public static String joinSampleName() {
return String.format("%s.%s = %s.%s",
"sn", COL_SN_ID,
"sv", COL_SV_SAMPLE_NAME_ID);
}
/**
* SQL fragment to join sample_value with label_set.
* Usage: FROM sample_value sv JOIN label_set ls ON ...
*/
public static String joinLabelSet() {
return String.format("%s.%s = %s.%s",
"ls", COL_LS_ID,
"sv", COL_SV_LABEL_SET_ID);
}
/**
* SQL fragment to join label_set with label_set_membership.
* Usage: JOIN label_set_membership lsm ON ...
*/
public static String joinLabelSetMembership() {
return String.format("%s.%s = %s.%s",
"lsm", COL_LSM_LABEL_SET_ID,
"ls", COL_LS_ID);
}
/**
* SQL fragment to join label_set_membership with label_key.
* Usage: JOIN label_key lk ON ...
*/
public static String joinLabelKey() {
return String.format("%s.%s = %s.%s",
"lk", COL_LK_ID,
"lsm", COL_LSM_LABEL_KEY_ID);
}
/**
* SQL fragment to join label_set_membership with label_value.
* Usage: JOIN label_value lv ON ...
*/
public static String joinLabelValue() {
return String.format("%s.%s = %s.%s",
"lv", COL_LV_ID,
"lsm", COL_LSM_LABEL_VALUE_ID);
}
/**
* Complete SQL fragment for joining all label-related tables.
* Produces: JOIN label_set ls ON ... LEFT JOIN label_set_membership lsm ON ... etc.
*/
public static String joinAllLabels() {
return String.format(
"JOIN %s ls ON %s " +
"LEFT JOIN %s lsm ON %s " +
"LEFT JOIN %s lk ON %s " +
"LEFT JOIN %s lv ON %s",
TABLE_LABEL_SET, joinLabelSet(),
TABLE_LABEL_SET_MEMBERSHIP, joinLabelSetMembership(),
TABLE_LABEL_KEY, joinLabelKey(),
TABLE_LABEL_VALUE, joinLabelValue()
);
}
private MetricsSchema() {
// Utility class - prevent instantiation
}
}
@@ -0,0 +1 @@
io.nosqlbench.nb.mql.MQLApp
@@ -0,0 +1,124 @@
# MetricsQL Query Examples
This directory contains executable examples demonstrating all query commands in the MetricsQL query engine. Each example is validated by automated tests to ensure the documentation stays accurate.
## Example Files by Theme
### Basic Queries
- **[INSTANT_QUERIES.md](INSTANT_QUERIES.md)** - Get current/latest values
- Basic instant queries
- Label filtering
- Error metrics
- Multi-dimensional filtering
- **[RANGE_QUERIES.md](RANGE_QUERIES.md)** - Time-series data retrieval
- Time-series over windows
- Historical data analysis
- Error tracking over time
### Analysis Queries (Coming Soon)
- **RATE_QUERIES.md** - Calculate per-second rates
- Counter rate calculations
- Change detection
- Throughput analysis
- **AGGREGATION_QUERIES.md** - Sum, average, min, max operations
- Grouping by labels
- Cross-dimensional aggregations
- Statistical summaries
- **QUANTILE_QUERIES.md** - Percentile analysis
- Latency percentiles (p50, p95, p99)
- Timer statistics
- Performance analysis
### Advanced Queries (Future)
- **RATIO_QUERIES.md** - Calculate error rates and ratios
- **TOPK_QUERIES.md** - Find top N values
- **BINARY_OPERATIONS.md** - Arithmetic between metrics
## Test Database: examples.db
All examples use `examples.db`, which contains metrics from a simplified e-commerce API:
**Metrics:**
- `api_requests_total` - Request counters by method, endpoint, and status
- `api_latency_ms` - API latency timers with percentiles
**Timeline:**
- 5 snapshots spanning 10 minutes (2025-10-23 10:00:00Z to 10:10:00Z)
- Simulates normal traffic → increased load → high load → recovery
**Label Dimensions:**
- `method`: {GET, POST}
- `endpoint`: {/users, /products}
- `status`: {200, 404, 500}
See [TEST_DATABASES.md](../TEST_DATABASES.md) for detailed schema information.
## How Examples Work
Each example follows this format:
```markdown
## Example N: Description
**Command:**
```
command --param value --other param
```
**Expected Output:**
```
[exact formatted table output]
```
**What it shows:** Explanation of the use case
```
The `ExampleDocumentationTest` class automatically:
1. Parses all example markdown files
2. Extracts commands and expected results
3. Executes each command against examples.db
4. Validates the actual output matches expectations
**This means documentation is always accurate and tested!**
## Running Examples Manually
Once the CLI is implemented, you can run these examples:
```bash
# Point to the examples database
export EXAMPLES_DB=src/test/resources/testdata/examples.db
# Run any example command
nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total
# With different output formats
nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total --format json
nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total --format csv
```
## Regenerating examples.db
If you modify the schema or need to update the example data:
```bash
mvn exec:java -pl nb-apis/nb-mql-api \
-Dexec.mainClass="io.nosqlbench.nb.mql.testdata.ExampleDataGenerator" \
-Dexec.classpathScope=test
```
## Contributing New Examples
When adding a new query command:
1. Add examples to the appropriate themed markdown file
2. Follow the established format with command, expected output, and explanation
3. Run the tests to verify: `mvn test -Dtest=ExampleDocumentationTest`
4. The tests will automatically validate your examples
---
*These examples are automatically tested and validated with every build.*
@@ -0,0 +1,130 @@
# Instant Queries
Get the most recent value for metrics. Instant queries return the latest snapshot for each unique label set.
**Database:** `examples.db`
---
## Example 1: Get All Latest Values
**Command:**
```
instant --metric api_requests_total
```
**Expected Output:**
```
timestamp | value | labels
----------------------------------------------------------------------------------
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
2025-10-23 10:10:00.0 | 8500.0 | endpoint=/products, method=GET, status=200
2025-10-23 10:10:00.0 | 1500.0 | endpoint=/users, method=POST, status=200
2025-10-23 10:10:00.0 | 75.0 | endpoint=/users, method=GET, status=404
2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500
5 rows (Xms)
```
**What it shows:** Retrieves the latest value for all label combinations of a metric. Useful for dashboards showing current state.
**Use cases:**
- Current request counts by endpoint
- Latest error counts
- Current gauge values
---
## Example 2: Filter by Specific Labels
**Command:**
```
instant --metric api_requests_total --labels method=GET,endpoint=/users,status=200
```
**Expected Output:**
```
timestamp | value | labels
----------------------------------------------------------------------------------
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
1 row (Xms)
```
**What it shows:** Filtering to a specific label combination returns a single time series.
**Use cases:**
- Monitor specific endpoint traffic
- Track particular operation types
- Isolate specific service behavior
---
## Example 3: Query Only Errors
**Command:**
```
instant --metric api_requests_total --labels status=500
```
**Expected Output:**
```
timestamp | value | labels
--------------------------------------------------------------------------------
2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500
1 row (Xms)
```
**What it shows:** Filter by a single label dimension (status code) to see current error count.
**Use cases:**
- Current error count for alerting
- Quick health check
- SLA monitoring
---
## Example 4: Filter Across Multiple Series
**Command:**
```
instant --metric api_requests_total --labels status=200
```
**Expected Output:**
```
timestamp | value | labels
----------------------------------------------------------------------------------
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
2025-10-23 10:10:00.0 | 8500.0 | endpoint=/products, method=GET, status=200
2025-10-23 10:10:00.0 | 1500.0 | endpoint=/users, method=POST, status=200
3 rows (Xms)
```
**What it shows:** Filtering by a shared label value returns all matching label sets. This groups all successful requests while preserving endpoint/method distinctions.
**Use cases:**
- All successful requests across endpoints
- All error types (status >= 400)
- Traffic patterns by method type
---
## Key Concepts
**Label Filtering:**
- Specify labels as comma-separated key=value pairs
- All specified labels must match (AND logic)
- Unspecified labels can have any value
**Output:**
- `timestamp`: When the snapshot was captured
- `value`: The metric value at that time
- `labels`: All labels for this time series (comma-separated)
**Performance:**
- Instant queries are fast (typically <5ms)
- Queries only the latest snapshot
- Efficient for real-time dashboards
@@ -0,0 +1,93 @@
# Range Queries
Retrieve time-series data over a time window. Range queries return all data points within a specified time range.
**Database:** `examples.db`
---
## Example 5: Time Series - Full History
**Command:**
```
range --metric api_requests_total --window 1h --labels method=GET,endpoint=/users,status=200
```
**Expected Output:**
```
timestamp | value | labels
----------------------------------------------------------------------------------
2025-10-23 10:00:00.0 | 0.0 | endpoint=/users, method=GET, status=200
2025-10-23 10:01:00.0 | 1000.0 | endpoint=/users, method=GET, status=200
2025-10-23 10:02:00.0 | 2200.0 | endpoint=/users, method=GET, status=200
2025-10-23 10:05:00.0 | 5500.0 | endpoint=/users, method=GET, status=200
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
5 rows (Xms)
```
**What it shows:** Complete time-series progression showing how a counter grew from 0 to 11,000 requests over 10 minutes.
**Use cases:**
- Visualize traffic patterns over time
- Analyze growth trends
- Export to graphing tools
- Historical analysis
---
## Example 6: Track Errors Over Time
**Command:**
```
range --metric api_requests_total --window 1h --labels status=500
```
**Expected Output:**
```
timestamp | value | labels
--------------------------------------------------------------------------------
2025-10-23 10:00:00.0 | 0.0 | endpoint=/users, method=GET, status=500
2025-10-23 10:01:00.0 | 2.0 | endpoint=/users, method=GET, status=500
2025-10-23 10:02:00.0 | 4.0 | endpoint=/users, method=GET, status=500
2025-10-23 10:05:00.0 | 8.0 | endpoint=/users, method=GET, status=500
2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500
5 rows (Xms)
```
**What it shows:** Track how server errors accumulated during the session. Notice the accelerated error rate during the high-load period (10:02 to 10:05).
**Use cases:**
- Incident investigation
- Error rate trending
- Correlate errors with load spikes
- SLA violation detection
---
## Key Concepts
**Time Windows:**
- `--window 5m` - Last 5 minutes from latest snapshot
- `--window 1h` - Last hour
- `--last 10m` - Alternative syntax for --window
**Alternative: Explicit Time Range (Future):**
```
range --metric api_requests_total --start 1729681200000 --end 1729684800000
```
**Output Ordering:**
- Results are ordered by timestamp (oldest first)
- Within same timestamp, ordered by labels alphabetically
**Performance:**
- Range queries scan all data in the time window
- Use label filters to reduce result set
- Typical query time: <20ms for 1 hour of data
**Visualization:**
- Export to CSV for Excel/Sheets: `--format csv`
- Export to JSON for plotting libraries: `--format json`
- Table format for quick console analysis
@@ -0,0 +1,135 @@
# MetricsQL Query Engine
MetricsQL is a query engine for analyzing NoSQLBench SQLite metrics databases. It provides a MetricsQL/PromQL-inspired interface for querying time-series metrics data.
## Quick Start
```bash
# Show session information
nb5 mql session
# List available metrics
nb5 mql summary
# Get current metric values
nb5 mql instant --metric api_requests_total
# Query time-series data
nb5 mql range --metric ops_total --window 5m
# Calculate request rate
nb5 mql rate --metric requests_total --window 5m
```
## Available Commands
### Discovery Commands
- **session** - Display session timeline and snapshot information
- **summary** - List all metrics with types and sample counts
- **sql** - Execute raw SQLite queries for advanced analysis
### Query Commands
- **instant** - Get the most recent value for a metric
- **range** - Retrieve time-series data over a time window
- **topk** - Find top N label sets by metric value
### Calculation Commands
- **rate** - Calculate per-second rate of change for counters
- **increase** - Calculate total increase in counter over time window
- **quantile** - Extract percentile values from timer/summary metrics
- **agg** - Perform aggregations (sum, avg, max, min, count) with grouping
- **ratio** - Calculate ratio between two metrics (e.g., error rate)
## Output Formats
All commands support multiple output formats via `--format`:
- **table** (default) - ASCII tables for console
- **json** - Pretty-printed JSON for programmatic use
- **jsonl** - JSON Lines for streaming
- **csv** - Comma-separated values for spreadsheets
- **tsv** - Tab-separated values for Unix tools
- **markdown** - GitHub-flavored markdown tables
## Examples
See detailed examples organized by topic:
- [Examples Overview](mql-examples.md) - Navigation and all examples
- [Instant Queries](mql-instant-queries.md) - Point-in-time queries
- [Range Queries](mql-range-queries.md) - Time-series queries
- More examples coming for rate, aggregation, and quantile queries
## Database Location
By default, commands query `logs/metrics.db` which is a symlink to the current session's metrics database.
To query a specific session:
```bash
nb5 mql instant --db logs/session_20251023_100000_metrics.db --metric ops_total
```
## Common Patterns
### Monitoring
```bash
# Current error count
nb5 mql instant --metric api_requests_total --labels status=500
# Request rate last 5 minutes
nb5 mql rate --metric api_requests_total --window 5m --labels status=200
# p95 latency
nb5 mql quantile --metric api_latency --p 0.95
```
### Analysis
```bash
# Total requests by endpoint
nb5 mql agg --metric api_requests_total --func sum --by endpoint
# Top 10 busiest endpoints
nb5 mql topk --metric api_requests_total --n 10
# Error rate
nb5 mql ratio --numerator errors_total --denominator requests_total
```
### Exporting Data
```bash
# CSV for Excel
nb5 mql range --metric ops_total --window 1h --format csv > metrics.csv
# JSON for analysis
nb5 mql summary --format json > metrics_inventory.json
# Streaming JSONL
nb5 mql range --metric ops_total --window 1h --format jsonl | jq '.value'
```
## Advanced Usage
### SQL Queries
```bash
# Custom aggregation
nb5 mql sql --query "
SELECT sn.sample, AVG(sv.value) as avg_value
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
GROUP BY sn.sample
"
# Schema inspection
nb5 mql sql --query "PRAGMA table_info(sample_value)"
```
### Debug Mode
```bash
# Show generated SQL
nb5 mql instant --metric ops_total --explain
```
## Learn More
Run `nb5 help mql` to see this documentation.
For detailed examples, see the [examples README](docs/mql/README.md).
@@ -0,0 +1,229 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class AggregateCommandTest {
@Test
void testSumNoGrouping() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"function", "sum"
);
QueryResult result = command.execute(conn, params);
// Should return 1 row with total sum
assertEquals(1, result.rowCount());
assertEquals(1, result.columns().size());
assertEquals("sum", result.columns().get(0));
System.out.println("\n=== Aggregate: Sum (No Grouping) ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSumByActivity() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"function", "sum",
"by", List.of("activity")
);
QueryResult result = command.execute(conn, params);
// Should return 2 rows (read and write)
assertEquals(2, result.rowCount(), "Should have 2 activities");
assertEquals(2, result.columns().size());
assertTrue(result.columns().contains("activity"));
assertTrue(result.columns().contains("sum"));
System.out.println("\n=== Aggregate: Sum by Activity ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testAvgByRegion() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"function", "avg",
"by", List.of("region")
);
QueryResult result = command.execute(conn, params);
// Should return 2 rows (us-east and us-west)
assertEquals(2, result.rowCount());
assertTrue(result.columns().contains("region"));
assertTrue(result.columns().contains("avg"));
System.out.println("\n=== Aggregate: Avg by Region ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testMaxByHost() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"function", "max",
"by", List.of("host")
);
QueryResult result = command.execute(conn, params);
// Should return 3 rows (server1, server2, server3)
assertEquals(3, result.rowCount(), "Should have 3 hosts");
System.out.println("\n=== Aggregate: Max by Host ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testCountByMultipleLabels() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"function", "count",
"by", List.of("activity", "region")
);
QueryResult result = command.execute(conn, params);
// Should return 4 rows (2 activities * 2 regions)
assertEquals(4, result.rowCount(), "Should have 4 combinations");
assertEquals(3, result.columns().size());
assertTrue(result.columns().contains("activity"));
assertTrue(result.columns().contains("region"));
assertTrue(result.columns().contains("count"));
System.out.println("\n=== Aggregate: Count by Activity and Region ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSumWithLabelFilter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"function", "sum",
"labels", Map.of("env", "prod")
);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount());
System.out.println("\n=== Aggregate: Sum (env=prod only) ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSumExamplesDatabase() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
AggregateCommand command = new AggregateCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"function", "sum",
"by", List.of("status")
);
QueryResult result = command.execute(conn, params);
// Should return 3 rows (200, 404, 500)
assertEquals(3, result.rowCount());
System.out.println("\n=== Aggregate: Sum by Status Code ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testValidationMissingMetric() {
AggregateCommand command = new AggregateCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("function", "sum")));
}
@Test
void testValidationMissingFunction() {
AggregateCommand command = new AggregateCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test")));
}
@Test
void testValidationInvalidFunction() {
AggregateCommand command = new AggregateCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "function", "invalid")));
}
@Test
void testValidationValidFunctions() throws Exception {
AggregateCommand command = new AggregateCommand();
command.validate(Map.of("metric", "test", "function", "sum"));
command.validate(Map.of("metric", "test", "function", "avg"));
command.validate(Map.of("metric", "test", "function", "max"));
command.validate(Map.of("metric", "test", "function", "min"));
command.validate(Map.of("metric", "test", "function", "count"));
}
}
@@ -0,0 +1,186 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class IncreaseCommandTest {
@Test
void testIncreaseLinearPattern() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
IncreaseCommand command = new IncreaseCommand();
Map<String, Object> params = Map.of(
"metric", "patterns_total",
"window", "1h",
"labels", Map.of("pattern", "linear")
);
QueryResult result = command.execute(conn, params);
// Should return 1 row for the linear pattern label set
assertEquals(1, result.rowCount(), "Should have 1 result for linear pattern");
Map<String, Object> row = result.rows().get(0);
double increase = (double) row.get("increase");
// Linear pattern: 120 snapshots * 10 increment = 1190 total increase
assertEquals(1190.0, increase, 1.0, "Total increase should be ~1190");
System.out.println("\n=== Increase Query: Linear Pattern ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testIncreaseExponentialPattern() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
IncreaseCommand command = new IncreaseCommand();
Map<String, Object> params = Map.of(
"metric", "patterns_total",
"window", "1h",
"labels", Map.of("pattern", "exponential")
);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount());
Map<String, Object> row = result.rows().get(0);
double increase = (double) row.get("increase");
// Exponential: 100 * (1.01^120) ≈ 230
assertTrue(increase > 200, "Exponential increase should be > 200");
assertTrue(increase < 300, "Exponential increase should be < 300");
System.out.println("\n=== Increase Query: Exponential Pattern ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testIncreaseStepPattern() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
IncreaseCommand command = new IncreaseCommand();
Map<String, Object> params = Map.of(
"metric", "patterns_total",
"window", "1h",
"labels", Map.of("pattern", "step")
);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount());
Map<String, Object> row = result.rows().get(0);
double increase = (double) row.get("increase");
// Step pattern: starts at 100, ends at 1200 (12 steps * 100)
assertEquals(1100.0, increase, 1.0, "Step increase should be 1100");
System.out.println("\n=== Increase Query: Step Pattern ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testIncreaseExamplesDatabase() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
IncreaseCommand command = new IncreaseCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"window", "1h",
"labels", Map.of("method", "GET", "endpoint", "/users", "status", "200")
);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount());
Map<String, Object> row = result.rows().get(0);
double increase = (double) row.get("increase");
// From 0 to 11000
assertEquals(11000.0, increase, 0.1);
System.out.println("\n=== Increase Query: Examples DB ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testIncreaseMultipleLabels() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
IncreaseCommand command = new IncreaseCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"window", "1h",
"labels", Map.of("status", "200")
);
QueryResult result = command.execute(conn, params);
// Should return 3 label sets (GET /users, GET /products, POST /users)
assertEquals(3, result.rowCount(), "Should have 3 label sets with status=200");
System.out.println("\n=== Increase Query: All Successful Requests ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testValidationMissingMetric() {
IncreaseCommand command = new IncreaseCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("window", "5m")));
}
@Test
void testValidationMissingWindow() {
IncreaseCommand command = new IncreaseCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test")));
}
@Test
void testValidationBothWindowAndLast() {
IncreaseCommand command = new IncreaseCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "window", "5m", "last", "10m")));
}
}
@@ -0,0 +1,173 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class InstantCommandTest {
@Test
void testInstantQuerySimpleCounter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
InstantCommand command = new InstantCommand();
Map<String, Object> params = Map.of("metric", "activity_ops_total");
QueryResult result = command.execute(conn, params);
// Should return 2 rows (read and write activities)
assertEquals(2, result.rowCount(), "Expected 2 label sets");
// Check columns
assertEquals(3, result.columns().size());
assertTrue(result.columns().contains("timestamp"));
assertTrue(result.columns().contains("value"));
assertTrue(result.columns().contains("labels"));
// Verify we got both read and write
boolean hasRead = false;
boolean hasWrite = false;
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
double value = (double) row.get("value");
if (labels.contains("activity=read")) {
hasRead = true;
assertEquals(90.0, value, 0.01, "Expected final read value to be 90");
} else if (labels.contains("activity=write")) {
hasWrite = true;
assertEquals(135.0, value, 0.01, "Expected final write value to be 135");
}
}
assertTrue(hasRead, "Should have read activity");
assertTrue(hasWrite, "Should have write activity");
// Print formatted output for visual verification
System.out.println("\n=== Instant Query Result ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testInstantQueryWithLabelFilter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
InstantCommand command = new InstantCommand();
Map<String, Object> params = Map.of(
"metric", "activity_ops_total",
"labels", Map.of("activity", "read")
);
QueryResult result = command.execute(conn, params);
// Should return only 1 row (read activity)
assertEquals(1, result.rowCount(), "Expected 1 label set with activity=read filter");
Map<String, Object> row = result.rows().get(0);
String labels = (String) row.get("labels");
double value = (double) row.get("value");
assertTrue(labels.contains("activity=read"));
assertEquals(90.0, value, 0.01);
}
}
@Test
void testInstantQueryMultiDimensional() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
InstantCommand command = new InstantCommand();
Map<String, Object> params = Map.of("metric", "requests_total");
QueryResult result = command.execute(conn, params);
// Should return 24 rows (24 unique label combinations)
assertEquals(24, result.rowCount(), "Expected 24 unique label sets");
System.out.println("\n=== Multi-Dimensional Instant Query ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testInstantQueryWithMultipleLabelFilters() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
InstantCommand command = new InstantCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"labels", Map.of(
"activity", "write",
"region", "us-east",
"env", "prod"
)
);
QueryResult result = command.execute(conn, params);
// Should return only rows matching all 3 label filters
// With 3 hosts, should get 3 results
assertEquals(3, result.rowCount(),
"Expected 3 label sets (one per host) matching activity=write, region=us-east, env=prod");
// Verify all results have the correct labels
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
assertTrue(labels.contains("activity=write"));
assertTrue(labels.contains("region=us-east"));
assertTrue(labels.contains("env=prod"));
}
}
}
@Test
void testValidationMissingMetric() {
InstantCommand command = new InstantCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of()));
}
@Test
void testValidationEmptyMetric() {
InstantCommand command = new InstantCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "")));
}
@Test
void testValidationInvalidLabelsType() {
InstantCommand command = new InstantCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "labels", "not a map")));
}
}
@@ -0,0 +1,167 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class QuantileCommandTest {
@Test
void testQuantileP95AllOperations() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
QuantileCommand command = new QuantileCommand();
Map<String, Object> params = Map.of(
"metric", "operation_latency",
"quantile", 0.95
);
QueryResult result = command.execute(conn, params);
// Should have p95 values for all operations across all snapshots
assertTrue(result.rowCount() > 0, "Should have quantile results");
// Verify we have the expected columns
assertEquals(4, result.columns().size());
assertTrue(result.columns().contains("timestamp"));
assertTrue(result.columns().contains("quantile"));
assertTrue(result.columns().contains("quantile_value"));
assertTrue(result.columns().contains("labels"));
// All quantiles should be 0.95
for (Map<String, Object> row : result.rows()) {
double q = (double) row.get("quantile");
assertEquals(0.95, q, 0.001);
}
System.out.println("\n=== Quantile Query: p95 All Operations ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testQuantileP99SpecificOperation() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
QuantileCommand command = new QuantileCommand();
Map<String, Object> params = Map.of(
"metric", "operation_latency",
"quantile", 0.99,
"labels", Map.of("operation", "select")
);
QueryResult result = command.execute(conn, params);
assertTrue(result.rowCount() > 0, "Should have p99 for select operation");
// All should be for select operation
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
assertTrue(labels.contains("operation=select"));
}
System.out.println("\n=== Quantile Query: p99 Select Operation ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testQuantileP50Median() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
QuantileCommand command = new QuantileCommand();
Map<String, Object> params = Map.of(
"metric", "operation_latency",
"quantile", 0.50
);
QueryResult result = command.execute(conn, params);
assertTrue(result.rowCount() > 0, "Should have median values");
System.out.println("\n=== Quantile Query: p50 (Median) ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testQuantileMultipleOperationsP95() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
QuantileCommand command = new QuantileCommand();
Map<String, Object> params = Map.of(
"metric", "operation_latency",
"quantile", 0.95
);
QueryResult result = command.execute(conn, params);
// Should have results for all 4 operations (select, insert, update, delete)
// across 6 snapshots = 24 total
assertEquals(24, result.rowCount(), "Should have 24 quantile values (4 ops * 6 snapshots)");
}
}
@Test
void testValidationMissingMetric() {
QuantileCommand command = new QuantileCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("quantile", 0.95)));
}
@Test
void testValidationMissingQuantile() {
QuantileCommand command = new QuantileCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test")));
}
@Test
void testValidationQuantileOutOfRange() {
QuantileCommand command = new QuantileCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "quantile", -0.1)));
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "quantile", 1.5)));
}
@Test
void testValidationValidQuantiles() throws Exception {
QuantileCommand command = new QuantileCommand();
command.validate(Map.of("metric", "test", "quantile", 0.0));
command.validate(Map.of("metric", "test", "quantile", 0.5));
command.validate(Map.of("metric", "test", "quantile", 1.0));
}
}
@@ -0,0 +1,191 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class RangeCommandTest {
@Test
void testRangeQueryAllData() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RangeCommand command = new RangeCommand();
// Query all data using a very large window
Map<String, Object> params = Map.of(
"metric", "activity_ops_total",
"window", "1h"
);
QueryResult result = command.execute(conn, params);
// Should return all 20 samples (10 snapshots × 2 label sets)
assertEquals(20, result.rowCount(), "Should return all samples");
// Verify columns
assertEquals(3, result.columns().size());
assertTrue(result.columns().contains("timestamp"));
assertTrue(result.columns().contains("value"));
assertTrue(result.columns().contains("labels"));
System.out.println("\n=== Range Query All Data ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testRangeQueryWithLabelFilter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RangeCommand command = new RangeCommand();
Map<String, Object> params = Map.of(
"metric", "activity_ops_total",
"window", "1h",
"labels", Map.of("activity", "read")
);
QueryResult result = command.execute(conn, params);
// Should return 10 samples (just read activity)
assertEquals(10, result.rowCount(), "Should return only read activity samples");
// Verify all have activity=read
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
assertTrue(labels.contains("activity=read"), "All results should be activity=read");
assertFalse(labels.contains("activity=write"), "Should not contain write");
}
System.out.println("\n=== Range Query Filtered (read only) ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testRangeQueryVerifyTimeSeries() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RangeCommand command = new RangeCommand();
Map<String, Object> params = Map.of(
"metric", "activity_ops_total",
"window", "1h",
"labels", Map.of("activity", "read")
);
QueryResult result = command.execute(conn, params);
// Verify values form expected linear sequence: 0, 10, 20, ..., 90
List<Double> values = result.rows().stream()
.map(row -> (Double) row.get("value"))
.sorted()
.toList();
assertEquals(10, values.size());
for (int i = 0; i < 10; i++) {
assertEquals(i * 10.0, values.get(i), 0.01,
"Value at index " + i + " should be " + (i * 10));
}
}
}
@Test
void testRangeQueryExamplesDatabase() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RangeCommand command = new RangeCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"window", "1h", // Get all 5 snapshots
"labels", Map.of("method", "GET", "endpoint", "/users", "status", "200")
);
QueryResult result = command.execute(conn, params);
// Should return 5 snapshots
assertEquals(5, result.rowCount(), "Should return all 5 snapshots");
// Verify values are monotonically increasing: 0, 1000, 2200, 5500, 11000
List<Double> values = result.rows().stream()
.map(row -> (Double) row.get("value"))
.toList();
assertEquals(0.0, values.get(0), 0.01);
assertEquals(1000.0, values.get(1), 0.01);
assertEquals(2200.0, values.get(2), 0.01);
assertEquals(5500.0, values.get(3), 0.01);
assertEquals(11000.0, values.get(4), 0.01);
System.out.println("\n=== Range Query Examples DB ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testRangeQueryAlternativeLastSyntax() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RangeCommand command = new RangeCommand();
// Test --last as alternative to --window
Map<String, Object> params = Map.of(
"metric", "activity_ops_total",
"last", "1h"
);
QueryResult result = command.execute(conn, params);
assertEquals(20, result.rowCount(), "Should return all samples with --last");
}
}
@Test
void testValidationMissingMetric() {
RangeCommand command = new RangeCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("window", "5m")));
}
@Test
void testValidationMissingTimeRange() {
RangeCommand command = new RangeCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test")));
}
@Test
void testValidationMultipleTimeRangeMethods() {
RangeCommand command = new RangeCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "window", "5m", "last", "10m")));
}
}
@@ -0,0 +1,201 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class RateCommandTest {
@Test
void testRateLinearPattern() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RateCommand command = new RateCommand();
Map<String, Object> params = Map.of(
"metric", "patterns_total",
"window", "1h",
"labels", Map.of("pattern", "linear")
);
QueryResult result = command.execute(conn, params);
// Linear pattern: increments by 10 every ~50ms
// Expected rate: 10 / 0.05 = 200 ops/sec (approximately)
assertTrue(result.rowCount() > 0, "Should have rate calculations");
// Get all calculated rates
List<Double> rates = result.rows().stream()
.map(row -> (Double) row.get("rate_per_sec"))
.filter(rate -> rate != null && !rate.isNaN())
.toList();
assertTrue(rates.size() > 0, "Should have valid rates");
// For linear pattern, all rates should be relatively constant
// (within reasonable variance due to timing)
double avgRate = rates.stream().mapToDouble(Double::doubleValue).average().orElse(0);
assertTrue(avgRate > 0, "Average rate should be positive");
System.out.println("\n=== Rate Query: Linear Pattern ===");
System.out.println(new TableFormatter().format(result));
System.out.println("Average rate: " + avgRate + " ops/sec");
}
}
@Test
void testRateExponentialPattern() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RateCommand command = new RateCommand();
Map<String, Object> params = Map.of(
"metric", "patterns_total",
"window", "1h",
"labels", Map.of("pattern", "exponential")
);
QueryResult result = command.execute(conn, params);
assertTrue(result.rowCount() > 0, "Should have rate calculations");
// Exponential pattern: rate should increase over time
List<Double> rates = result.rows().stream()
.map(row -> (Double) row.get("rate_per_sec"))
.filter(rate -> rate != null && !rate.isNaN())
.toList();
assertTrue(rates.size() > 10, "Should have many rate points");
// Verify rates are generally increasing (exponential growth)
// Compare first quarter to last quarter
int quarterSize = rates.size() / 4;
double firstQuarterAvg = rates.subList(0, quarterSize).stream()
.mapToDouble(Double::doubleValue).average().orElse(0);
double lastQuarterAvg = rates.subList(rates.size() - quarterSize, rates.size()).stream()
.mapToDouble(Double::doubleValue).average().orElse(0);
assertTrue(lastQuarterAvg > firstQuarterAvg,
"Exponential pattern should have increasing rate over time");
System.out.println("\n=== Rate Query: Exponential Pattern ===");
System.out.println("First quarter avg: " + firstQuarterAvg + " ops/sec");
System.out.println("Last quarter avg: " + lastQuarterAvg + " ops/sec");
}
}
@Test
void testRateStepPattern() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RateCommand command = new RateCommand();
Map<String, Object> params = Map.of(
"metric", "patterns_total",
"window", "1h",
"labels", Map.of("pattern", "step")
);
QueryResult result = command.execute(conn, params);
assertTrue(result.rowCount() > 0, "Should have rate calculations");
// Step pattern: rate should be 0 within steps, high at step boundaries
List<Double> rates = result.rows().stream()
.map(row -> (Double) row.get("rate_per_sec"))
.filter(rate -> rate != null && !rate.isNaN())
.toList();
// Most rates should be either 0 (flat) or very high (step jump)
long zeroRates = rates.stream().filter(r -> Math.abs(r) < 1.0).count();
long highRates = rates.stream().filter(r -> r > 100.0).count();
assertTrue(zeroRates > 0 || highRates > 0,
"Step pattern should have either zero or high rates");
System.out.println("\n=== Rate Query: Step Pattern ===");
System.out.println("Zero rates: " + zeroRates + ", High rates: " + highRates);
}
}
@Test
void testRateExamplesDatabase() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
RateCommand command = new RateCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"window", "1h",
"labels", Map.of("method", "GET", "endpoint", "/users", "status", "200")
);
QueryResult result = command.execute(conn, params);
// Should have 4 rate calculations (5 snapshots - 1 for first point)
assertEquals(4, result.rowCount(), "Should have 4 rate points");
System.out.println("\n=== Rate Query: Examples DB ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testValidationMissingMetric() {
RateCommand command = new RateCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("window", "5m")));
}
@Test
void testValidationMissingWindow() {
RateCommand command = new RateCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test")));
}
@Test
void testValidationBothWindowAndLast() {
RateCommand command = new RateCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "window", "5m", "last", "10m")));
}
@Test
void testValidationWindowAllowed() throws Exception {
RateCommand command = new RateCommand();
command.validate(Map.of("metric", "test", "window", "5m"));
}
@Test
void testValidationLastAllowed() throws Exception {
RateCommand command = new RateCommand();
command.validate(Map.of("metric", "test", "last", "5m"));
}
}
@@ -0,0 +1,74 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class RatioCommandTest {
@Test
void testRatioCommandExists() throws Exception {
// Basic smoke test - just verify command can be created and validated
RatioCommand command = new RatioCommand();
assertEquals("ratio", command.getName());
// Valid params should pass validation
command.validate(Map.of(
"numerator", "metric1_total",
"denominator", "metric2_total"
));
}
@Test
void testValidationMissingNumerator() {
RatioCommand command = new RatioCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("denominator", "total")));
}
@Test
void testValidationMissingDenominator() {
RatioCommand command = new RatioCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("numerator", "errors")));
}
@Test
void testValidationEmptyMetrics() {
RatioCommand command = new RatioCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("numerator", "", "denominator", "total")));
}
@Test
void testValidationValid() throws Exception {
RatioCommand command = new RatioCommand();
command.validate(Map.of("numerator", "errors", "denominator", "total"));
}
}
@@ -0,0 +1,91 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class SessionCommandTest {
@Test
void testSessionInfo() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SessionCommand command = new SessionCommand();
QueryResult result = command.execute(conn, Map.of());
// Should return 1 row with session info
assertEquals(1, result.rowCount());
// Verify columns
assertEquals(6, result.columns().size());
assertTrue(result.columns().contains("first_snapshot"));
assertTrue(result.columns().contains("last_snapshot"));
assertTrue(result.columns().contains("duration"));
assertTrue(result.columns().contains("total_snapshots"));
assertTrue(result.columns().contains("total_samples"));
assertTrue(result.columns().contains("avg_interval"));
Map<String, Object> row = result.rows().get(0);
// Verify we have 5 snapshots
int snapshots = (int) row.get("total_snapshots");
assertEquals(5, snapshots, "Should have 5 snapshots");
// Should have 35 total samples
int samples = (int) row.get("total_samples");
assertEquals(35, samples, "Should have 35 samples");
System.out.println("\n=== Session Command ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSessionInfoSimpleCounter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SessionCommand command = new SessionCommand();
QueryResult result = command.execute(conn, Map.of());
assertEquals(1, result.rowCount());
Map<String, Object> row = result.rows().get(0);
int snapshots = (int) row.get("total_snapshots");
int samples = (int) row.get("total_samples");
assertEquals(10, snapshots, "Should have 10 snapshots");
assertEquals(20, samples, "Should have 20 samples");
System.out.println("\n=== Session Info: Simple Counter DB ===");
System.out.println(new TableFormatter().format(result));
}
}
}
@@ -0,0 +1,206 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class SqlCommandTest {
@Test
void testSqlQueryMetricFamilies() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SqlCommand command = new SqlCommand();
Map<String, Object> params = Map.of(
"query", "SELECT name, type FROM metric_family ORDER BY name"
);
QueryResult result = command.execute(conn, params);
// Should have at least 1 metric family
assertTrue(result.rowCount() >= 1, "Should have metric families");
// Verify columns
assertEquals(2, result.columns().size());
assertTrue(result.columns().contains("name"));
assertTrue(result.columns().contains("type"));
System.out.println("\n=== SQL Query: Metric Families ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSqlQuerySampleCounts() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SqlCommand command = new SqlCommand();
Map<String, Object> params = Map.of(
"query", "SELECT COUNT(*) as total_samples FROM sample_value"
);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount());
assertEquals(1, result.columns().size());
assertEquals("total_samples", result.columns().get(0));
Map<String, Object> row = result.rows().get(0);
long count = ((Number) row.get("total_samples")).longValue();
assertEquals(20, count, "Should have 20 samples");
System.out.println("\n=== SQL Query: Sample Count ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSqlQueryWithCTE() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SqlCommand command = new SqlCommand();
String sql = """
WITH latest AS (
SELECT MAX(timestamp_ms) as max_ts FROM sample_value
)
SELECT COUNT(*) as latest_count
FROM sample_value sv, latest
WHERE sv.timestamp_ms = latest.max_ts
""";
Map<String, Object> params = Map.of("query", sql);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount());
long count = ((Number) result.rows().get(0).get("latest_count")).longValue();
assertEquals(2, count, "Should have 2 samples at latest timestamp");
}
}
@Test
void testSqlQueryPragma() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SqlCommand command = new SqlCommand();
Map<String, Object> params = Map.of(
"query", "PRAGMA table_info(metric_family)"
);
QueryResult result = command.execute(conn, params);
// Should return schema information
assertTrue(result.rowCount() > 0, "Should return table info");
System.out.println("\n=== SQL Query: Table Schema ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testValidationMissingQuery() {
SqlCommand command = new SqlCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of()));
}
@Test
void testValidationEmptyQuery() {
SqlCommand command = new SqlCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("query", "")));
}
@Test
void testValidationNonSelectQuery() {
SqlCommand command = new SqlCommand();
// INSERT should be blocked
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("query", "INSERT INTO metric_family ...")));
// UPDATE should be blocked
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("query", "UPDATE sample_value SET ...")));
// DELETE should be blocked
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("query", "DELETE FROM sample_value")));
// DROP should be blocked
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("query", "DROP TABLE metric_family")));
}
@Test
void testValidationAllowedQueries() throws Exception {
SqlCommand command = new SqlCommand();
// SELECT should be allowed
command.validate(Map.of("query", "SELECT * FROM metric_family"));
// WITH (CTE) should be allowed
command.validate(Map.of("query", "WITH cte AS (SELECT 1) SELECT * FROM cte"));
// PRAGMA should be allowed
command.validate(Map.of("query", "PRAGMA table_info(metric_family)"));
}
@Test
void testSqlQueryExamples() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SqlCommand command = new SqlCommand();
// Example: Get all metrics with their latest values
String sql = """
SELECT
sn.sample,
sv.value,
datetime(sv.timestamp_ms/1000, 'unixepoch') as captured_at
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
WHERE sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
ORDER BY sn.sample, sv.value DESC
""";
Map<String, Object> params = Map.of("query", sql);
QueryResult result = command.execute(conn, params);
assertTrue(result.rowCount() > 0, "Should return results");
System.out.println("\n=== SQL Query: All Latest Values ===");
System.out.println(new TableFormatter().format(result));
}
}
}
@@ -0,0 +1,93 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class SummaryCommandTest {
@Test
void testSummaryExamplesDb() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SummaryCommand command = new SummaryCommand();
QueryResult result = command.execute(conn, Map.of());
// Should have multiple sections
assertTrue(result.rowCount() > 0, "Should have summary rows");
// Verify columns
assertEquals(4, result.columns().size());
assertTrue(result.columns().contains("section"));
assertTrue(result.columns().contains("metric"));
assertTrue(result.columns().contains("value"));
assertTrue(result.columns().contains("details"));
// Should have SESSION section
boolean hasSession = result.rows().stream()
.anyMatch(row -> "SESSION".equals(row.get("section")));
assertTrue(hasSession, "Should have SESSION section");
System.out.println("\n=== Summary: Examples DB ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSummarySimpleCounter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SummaryCommand command = new SummaryCommand();
QueryResult result = command.execute(conn, Map.of());
assertTrue(result.rowCount() > 0, "Should have summary rows");
System.out.println("\n=== Summary: Simple Counter DB ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testSummaryMultiDimensional() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
SummaryCommand command = new SummaryCommand();
QueryResult result = command.execute(conn, Map.of());
assertTrue(result.rowCount() > 0, "Should have summary rows");
System.out.println("\n=== Summary: Multi-Dimensional DB ===");
System.out.println(new TableFormatter().format(result));
}
}
}
@@ -0,0 +1,178 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.commands;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.InvalidQueryException;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class TopKCommandTest {
@Test
void testTop3() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
TopKCommand command = new TopKCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"n", 3
);
QueryResult result = command.execute(conn, params);
// Should return exactly 3 rows
assertEquals(3, result.rowCount(), "Should return top 3");
// Verify columns
assertEquals(2, result.columns().size());
assertTrue(result.columns().contains("value"));
assertTrue(result.columns().contains("labels"));
// First row should have highest value (11000)
double firstValue = (double) result.rows().get(0).get("value");
assertEquals(11000.0, firstValue, 0.1, "Top value should be 11000");
System.out.println("\n=== TopK Query: Top 3 ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testTop1() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
TopKCommand command = new TopKCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"n", 1
);
QueryResult result = command.execute(conn, params);
assertEquals(1, result.rowCount(), "Should return top 1");
double value = (double) result.rows().get(0).get("value");
assertEquals(11000.0, value, 0.1);
System.out.println("\n=== TopK Query: Top 1 ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testTop5WithLabelFilter() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
TopKCommand command = new TopKCommand();
Map<String, Object> params = Map.of(
"metric", "api_requests_total",
"n", 5,
"labels", Map.of("status", "200")
);
QueryResult result = command.execute(conn, params);
// Should return 3 rows (only 3 label sets with status=200)
assertEquals(3, result.rowCount(), "Should return 3 (all status=200)");
// All should have status=200
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
assertTrue(labels.contains("status=200"));
}
System.out.println("\n=== TopK Query: Top 5 (filtered by status=200) ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testTop10MultiDimensional() throws Exception {
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
TopKCommand command = new TopKCommand();
Map<String, Object> params = Map.of(
"metric", "requests_total",
"n", 10
);
QueryResult result = command.execute(conn, params);
// Should return exactly 10 rows
assertEquals(10, result.rowCount(), "Should return top 10");
// Verify values are in descending order
double previousValue = Double.MAX_VALUE;
for (Map<String, Object> row : result.rows()) {
double value = (double) row.get("value");
assertTrue(value <= previousValue, "Values should be in descending order");
previousValue = value;
}
System.out.println("\n=== TopK Query: Top 10 from Multi-Dimensional ===");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testValidationMissingMetric() {
TopKCommand command = new TopKCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("n", 5)));
}
@Test
void testValidationMissingN() {
TopKCommand command = new TopKCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test")));
}
@Test
void testValidationNegativeN() {
TopKCommand command = new TopKCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "n", -1)));
}
@Test
void testValidationZeroN() {
TopKCommand command = new TopKCommand();
assertThrows(InvalidQueryException.class, () ->
command.validate(Map.of("metric", "test", "n", 0)));
}
@Test
void testValidationValidN() throws Exception {
TopKCommand command = new TopKCommand();
command.validate(Map.of("metric", "test", "n", 1));
command.validate(Map.of("metric", "test", "n", 100));
}
}
@@ -0,0 +1,347 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.integration;
import io.nosqlbench.nb.mql.commands.InstantCommand;
import io.nosqlbench.nb.mql.commands.RangeCommand;
import io.nosqlbench.nb.mql.format.TableFormatter;
import io.nosqlbench.nb.mql.query.QueryResult;
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.file.Path;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.*;
/**
* Validates that the examples in EXAMPLES.md produce the expected output.
* This test parses the markdown file, executes the commands, and verifies
* the results match the documented expectations.
*/
class ExampleDocumentationTest {
private static Path examplesDb;
private static List<Example> examples;
@BeforeAll
static void loadExamples() throws IOException {
// Load the examples database
examplesDb = TestDatabaseLoader.getDatabase("examples.db");
// Parse all example markdown files from classpath
examples = new ArrayList<>();
String[] exampleFiles = {
"mql-instant-queries.md",
"mql-range-queries.md"
// Add more as they are created:
// "mql-rate-queries.md",
// "mql-aggregation-queries.md",
// etc.
};
for (String exampleFile : exampleFiles) {
String content = new String(
ExampleDocumentationTest.class.getClassLoader()
.getResourceAsStream(exampleFile)
.readAllBytes()
);
examples.addAll(parseExamples(content));
System.out.println("Loaded examples from " + exampleFile);
}
System.out.println("Total: " + examples.size() + " examples loaded");
}
@Test
void testExample1_InstantQueryAllLabels() throws Exception {
Example example = findExample("Example 1");
assertNotNull(example, "Example 1 should exist");
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
QueryResult result = executeCommand(conn, example.command);
// Verify row count
assertEquals(5, result.rowCount(), "Example 1 should return 5 label sets");
// Verify all values are present in the result
Map<String, Double> expectedValues = Map.of(
"endpoint=/users, method=GET, status=200", 11000.0,
"endpoint=/products, method=GET, status=200", 8500.0,
"endpoint=/users, method=POST, status=200", 1500.0,
"endpoint=/users, method=GET, status=404", 75.0,
"endpoint=/users, method=GET, status=500", 15.0
);
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
double value = (double) row.get("value");
boolean matched = false;
for (Map.Entry<String, Double> expected : expectedValues.entrySet()) {
if (labelsMatch(labels, expected.getKey())) {
assertEquals(expected.getValue(), value, 0.01,
"Value mismatch for labels: " + labels);
matched = true;
break;
}
}
assertTrue(matched, "Unexpected label set: " + labels);
}
System.out.println("Example 1 output:");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testExample2_InstantQueryWithFilter() throws Exception {
Example example = findExample("Example 2");
assertNotNull(example, "Example 2 should exist");
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
QueryResult result = executeCommand(conn, example.command);
// Should return exactly 1 row
assertEquals(1, result.rowCount(), "Filtered query should return 1 row");
Map<String, Object> row = result.rows().get(0);
double value = (double) row.get("value");
String labels = (String) row.get("labels");
assertEquals(11000.0, value, 0.01);
assertTrue(labels.contains("method=GET"));
assertTrue(labels.contains("endpoint=/users"));
assertTrue(labels.contains("status=200"));
System.out.println("Example 2 output:");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testExample3_InstantQueryErrors() throws Exception {
Example example = findExample("Example 3");
assertNotNull(example, "Example 3 should exist");
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
QueryResult result = executeCommand(conn, example.command);
assertEquals(1, result.rowCount(), "Should return 1 error metric");
Map<String, Object> row = result.rows().get(0);
double value = (double) row.get("value");
String labels = (String) row.get("labels");
assertEquals(15.0, value, 0.01);
assertTrue(labels.contains("status=500"));
System.out.println("Example 3 output:");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testExample4_InstantQuerySuccessful() throws Exception {
Example example = findExample("Example 4");
assertNotNull(example, "Example 4 should exist");
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
QueryResult result = executeCommand(conn, example.command);
assertEquals(3, result.rowCount(), "Should return 3 successful request types");
// Verify all have status=200
for (Map<String, Object> row : result.rows()) {
String labels = (String) row.get("labels");
assertTrue(labels.contains("status=200"), "All results should have status=200");
}
System.out.println("Example 4 output:");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testExample5_RangeQueryTimeSeries() throws Exception {
Example example = findExample("Example 5");
assertNotNull(example, "Example 5 should exist");
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
QueryResult result = executeCommand(conn, example.command);
// Should return 5 snapshots for the single label set
assertEquals(5, result.rowCount(), "Should return 5 time-series points");
// Verify values are: 0, 1000, 2200, 5500, 11000
List<Double> values = result.rows().stream()
.map(row -> (Double) row.get("value"))
.toList();
assertEquals(0.0, values.get(0), 0.01);
assertEquals(1000.0, values.get(1), 0.01);
assertEquals(2200.0, values.get(2), 0.01);
assertEquals(5500.0, values.get(3), 0.01);
assertEquals(11000.0, values.get(4), 0.01);
System.out.println("Example 5 output:");
System.out.println(new TableFormatter().format(result));
}
}
@Test
void testExample6_RangeQueryErrors() throws Exception {
Example example = findExample("Example 6");
assertNotNull(example, "Example 6 should exist");
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
QueryResult result = executeCommand(conn, example.command);
// Should return 5 snapshots for error metric
assertEquals(5, result.rowCount(), "Should return 5 time-series points for errors");
// Verify error values: 0, 2, 4, 8, 15
List<Double> values = result.rows().stream()
.map(row -> (Double) row.get("value"))
.toList();
assertEquals(0.0, values.get(0), 0.01);
assertEquals(2.0, values.get(1), 0.01);
assertEquals(4.0, values.get(2), 0.01);
assertEquals(8.0, values.get(3), 0.01);
assertEquals(15.0, values.get(4), 0.01);
System.out.println("Example 6 output:");
System.out.println(new TableFormatter().format(result));
}
}
// Helper methods
private static QueryResult executeCommand(Connection conn, String commandStr) throws Exception {
// Parse command string: "instant --metric <name> [--labels key=val,...]"
Map<String, Object> params = parseCommandParams(commandStr);
if (commandStr.startsWith("instant")) {
InstantCommand command = new InstantCommand();
return command.execute(conn, params);
} else if (commandStr.startsWith("range")) {
RangeCommand command = new RangeCommand();
return command.execute(conn, params);
}
throw new UnsupportedOperationException("Command not yet implemented: " + commandStr);
}
private static Map<String, Object> parseCommandParams(String commandStr) {
Map<String, Object> params = new LinkedHashMap<>();
// Extract --metric value
Pattern metricPattern = Pattern.compile("--metric\\s+(\\S+)");
Matcher metricMatcher = metricPattern.matcher(commandStr);
if (metricMatcher.find()) {
params.put("metric", metricMatcher.group(1));
}
// Extract --window value
Pattern windowPattern = Pattern.compile("--window\\s+(\\S+)");
Matcher windowMatcher = windowPattern.matcher(commandStr);
if (windowMatcher.find()) {
params.put("window", windowMatcher.group(1));
}
// Extract --last value
Pattern lastPattern = Pattern.compile("--last\\s+(\\S+)");
Matcher lastMatcher = lastPattern.matcher(commandStr);
if (lastMatcher.find()) {
params.put("last", lastMatcher.group(1));
}
// Extract --labels key=val,key=val
Pattern labelsPattern = Pattern.compile("--labels\\s+(\\S+)");
Matcher labelsMatcher = labelsPattern.matcher(commandStr);
if (labelsMatcher.find()) {
String labelsStr = labelsMatcher.group(1);
Map<String, String> labels = new LinkedHashMap<>();
for (String pair : labelsStr.split(",")) {
String[] kv = pair.split("=", 2);
if (kv.length == 2) {
labels.put(kv[0], kv[1]);
}
}
params.put("labels", labels);
}
return params;
}
private static boolean labelsMatch(String actual, String expected) {
// Normalize: remove spaces, sort key=value pairs
String[] actualPairs = actual.replace(" ", "").split(",");
String[] expectedPairs = expected.replace(" ", "").split(",");
if (actualPairs.length != expectedPairs.length) {
return false;
}
java.util.Arrays.sort(actualPairs);
java.util.Arrays.sort(expectedPairs);
return java.util.Arrays.equals(actualPairs, expectedPairs);
}
private static Example findExample(String exampleName) {
for (Example example : examples) {
if (example.name.contains(exampleName)) {
return example;
}
}
return null;
}
private static List<Example> parseExamples(String markdown) {
List<Example> result = new ArrayList<>();
// Pattern to match example sections
Pattern examplePattern = Pattern.compile(
"##\\s+(Example \\d+:[^\\n]+).*?" +
"\\*\\*Command:\\*\\*\\s*```([^`]+)```",
Pattern.DOTALL
);
Matcher matcher = examplePattern.matcher(markdown);
while (matcher.find()) {
String name = matcher.group(1).trim();
String command = matcher.group(2).trim();
result.add(new Example(name, command));
}
return result;
}
private record Example(String name, String command) {}
}
@@ -0,0 +1,141 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.query;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
class TimeWindowParserTest {
@Test
void testParseSeconds() throws InvalidQueryException {
assertEquals(5000L, TimeWindowParser.parseToMillis("5s"));
assertEquals(30000L, TimeWindowParser.parseToMillis("30s"));
assertEquals(1000L, TimeWindowParser.parseToMillis("1s"));
}
@Test
void testParseMinutes() throws InvalidQueryException {
assertEquals(60000L, TimeWindowParser.parseToMillis("1m"));
assertEquals(300000L, TimeWindowParser.parseToMillis("5m"));
assertEquals(900000L, TimeWindowParser.parseToMillis("15m"));
}
@Test
void testParseHours() throws InvalidQueryException {
assertEquals(3600000L, TimeWindowParser.parseToMillis("1h"));
assertEquals(7200000L, TimeWindowParser.parseToMillis("2h"));
assertEquals(86400000L, TimeWindowParser.parseToMillis("24h"));
}
@Test
void testParseDays() throws InvalidQueryException {
assertEquals(86400000L, TimeWindowParser.parseToMillis("1d"));
assertEquals(604800000L, TimeWindowParser.parseToMillis("7d"));
}
@Test
void testParseWeeks() throws InvalidQueryException {
assertEquals(604800000L, TimeWindowParser.parseToMillis("1w"));
assertEquals(1209600000L, TimeWindowParser.parseToMillis("2w"));
}
@Test
void testParseYears() throws InvalidQueryException {
assertEquals(31536000000L, TimeWindowParser.parseToMillis("1y"));
}
@Test
void testParseDecimal() throws InvalidQueryException {
assertEquals(1500L, TimeWindowParser.parseToMillis("1.5s"));
assertEquals(90000L, TimeWindowParser.parseToMillis("1.5m"));
}
@Test
void testCaseInsensitive() throws InvalidQueryException {
assertEquals(5000L, TimeWindowParser.parseToMillis("5S"));
assertEquals(300000L, TimeWindowParser.parseToMillis("5M"));
assertEquals(3600000L, TimeWindowParser.parseToMillis("1H"));
}
@Test
void testWhitespace() throws InvalidQueryException {
assertEquals(5000L, TimeWindowParser.parseToMillis(" 5s "));
assertEquals(300000L, TimeWindowParser.parseToMillis("\t5m\n"));
}
@Test
void testInvalidFormat() {
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis("5"));
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis("s"));
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis("5x"));
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis("five seconds"));
}
@Test
void testNullOrEmpty() {
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis(null));
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis(""));
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis(" "));
}
@Test
void testNegativeValue() {
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis("-5s"));
}
@Test
void testZeroValue() {
assertThrows(InvalidQueryException.class, () ->
TimeWindowParser.parseToMillis("0s"));
}
@Test
void testFormatMillis() {
assertEquals("5s", TimeWindowParser.formatMillis(5000L));
assertEquals("5m", TimeWindowParser.formatMillis(300000L));
assertEquals("1h", TimeWindowParser.formatMillis(3600000L));
assertEquals("1d", TimeWindowParser.formatMillis(86400000L));
assertEquals("1w", TimeWindowParser.formatMillis(604800000L));
}
@Test
void testFormatMillisNonExact() {
// Should fall back to smallest unit that works
assertEquals("1500ms", TimeWindowParser.formatMillis(1500L));
assertEquals("90s", TimeWindowParser.formatMillis(90000L));
}
@Test
void testRoundTrip() throws InvalidQueryException {
String[] inputs = {"5s", "10m", "2h", "1d", "1w"};
for (String input : inputs) {
long millis = TimeWindowParser.parseToMillis(input);
String formatted = TimeWindowParser.formatMillis(millis);
assertEquals(input, formatted, "Round trip failed for: " + input);
}
}
}
@@ -0,0 +1,89 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.schema;
import org.junit.jupiter.api.Test;
import java.util.LinkedHashMap;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.*;
class LabelSetResolverTest {
@Test
void testEmptyLabels() {
assertEquals("{}", LabelSetResolver.computeHash(Map.of()));
assertEquals("{}", LabelSetResolver.computeHash(null));
}
@Test
void testSingleLabel() {
Map<String, String> labels = Map.of("activity", "write");
assertEquals("{activity=write}", LabelSetResolver.computeHash(labels));
}
@Test
void testMultipleLabels() {
Map<String, String> labels = Map.of(
"activity", "write",
"host", "server1",
"region", "us-east"
);
// Should be sorted alphabetically by key
assertEquals("{activity=write,host=server1,region=us-east}",
LabelSetResolver.computeHash(labels));
}
@Test
void testLabelsSortedAlphabetically() {
// Insert in reverse order to test sorting
Map<String, String> labels = new LinkedHashMap<>();
labels.put("z_last", "value3");
labels.put("b_second", "value2");
labels.put("a_first", "value1");
String hash = LabelSetResolver.computeHash(labels);
assertEquals("{a_first=value1,b_second=value2,z_last=value3}", hash);
}
@Test
void testSameLabelsDifferentOrder() {
Map<String, String> labels1 = new LinkedHashMap<>();
labels1.put("activity", "write");
labels1.put("host", "server1");
Map<String, String> labels2 = new LinkedHashMap<>();
labels2.put("host", "server1");
labels2.put("activity", "write");
// Should produce same hash regardless of insertion order
assertEquals(LabelSetResolver.computeHash(labels1),
LabelSetResolver.computeHash(labels2));
}
@Test
void testSpecialCharacters() {
Map<String, String> labels = Map.of(
"label", "value=with=equals",
"another", "value,with,commas"
);
String hash = LabelSetResolver.computeHash(labels);
assertTrue(hash.contains("label=value=with=equals"));
assertTrue(hash.contains("another=value,with,commas"));
}
}
@@ -0,0 +1,109 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.testdata;
import java.nio.file.Path;
/**
* Generates the reference example database used in EXAMPLES.md documentation.
* This database has specific, well-known timestamps and values that are referenced
* in documentation and validated by ExampleDocumentationTest.
*
* The data represents a simplified e-commerce API with:
* - Request counters (GET/POST operations on different endpoints)
* - API latency timers
* - HTTP status codes (200, 404, 500)
*
* Timeline:
* - 2025-10-23 10:00:00Z - T+0: Initial state (zeros)
* - 2025-10-23 10:01:00Z - T+1min: Normal traffic
* - 2025-10-23 10:02:00Z - T+2min: Increased load
* - 2025-10-23 10:05:00Z - T+5min: High load period
* - 2025-10-23 10:10:00Z - T+10min: Back to normal
*/
public class ExampleDataGenerator {
/**
* Generate examples.db - the reference database for documentation examples.
*/
public static void generateExamples(Path outputPath) throws Exception {
MetricsDataBuilder.create(outputPath)
// T+0: Initial state (10:00:00)
.atTime("2025-10-23T10:00:00Z")
.counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "200")
.counter("api_requests", 0, "method", "GET", "endpoint", "/products", "status", "200")
.counter("api_requests", 0, "method", "POST", "endpoint", "/users", "status", "200")
.counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "404")
.counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "500")
.timer("api_latency", new long[]{10, 12, 15, 20, 25}, "method", "GET", "endpoint", "/users")
.timer("api_latency", new long[]{8, 10, 12, 14, 16}, "method", "GET", "endpoint", "/products")
// T+1min: Normal traffic (10:01:00)
.atTime("2025-10-23T10:01:00Z")
.counter("api_requests", 1000, "method", "GET", "endpoint", "/users", "status", "200")
.counter("api_requests", 800, "method", "GET", "endpoint", "/products", "status", "200")
.counter("api_requests", 150, "method", "POST", "endpoint", "/users", "status", "200")
.counter("api_requests", 5, "method", "GET", "endpoint", "/users", "status", "404")
.counter("api_requests", 2, "method", "GET", "endpoint", "/users", "status", "500")
.timer("api_latency", new long[]{10, 12, 15, 18, 22, 25, 30}, "method", "GET", "endpoint", "/users")
.timer("api_latency", new long[]{8, 10, 11, 13, 15, 18}, "method", "GET", "endpoint", "/products")
// T+2min: Increased load (10:02:00)
.atTime("2025-10-23T10:02:00Z")
.counter("api_requests", 2200, "method", "GET", "endpoint", "/users", "status", "200")
.counter("api_requests", 1650, "method", "GET", "endpoint", "/products", "status", "200")
.counter("api_requests", 320, "method", "POST", "endpoint", "/users", "status", "200")
.counter("api_requests", 12, "method", "GET", "endpoint", "/users", "status", "404")
.counter("api_requests", 4, "method", "GET", "endpoint", "/users", "status", "500")
.timer("api_latency", new long[]{12, 15, 18, 22, 28, 35, 42}, "method", "GET", "endpoint", "/users")
.timer("api_latency", new long[]{9, 11, 13, 15, 18, 22}, "method", "GET", "endpoint", "/products")
// T+5min: High load period (10:05:00)
.atTime("2025-10-23T10:05:00Z")
.counter("api_requests", 5500, "method", "GET", "endpoint", "/users", "status", "200")
.counter("api_requests", 4200, "method", "GET", "endpoint", "/products", "status", "200")
.counter("api_requests", 750, "method", "POST", "endpoint", "/users", "status", "200")
.counter("api_requests", 35, "method", "GET", "endpoint", "/users", "status", "404")
.counter("api_requests", 8, "method", "GET", "endpoint", "/users", "status", "500")
.timer("api_latency", new long[]{15, 20, 25, 32, 45, 58, 75}, "method", "GET", "endpoint", "/users")
.timer("api_latency", new long[]{10, 13, 16, 20, 25, 30}, "method", "GET", "endpoint", "/products")
// T+10min: Back to normal (10:10:00)
.atTime("2025-10-23T10:10:00Z")
.counter("api_requests", 11000, "method", "GET", "endpoint", "/users", "status", "200")
.counter("api_requests", 8500, "method", "GET", "endpoint", "/products", "status", "200")
.counter("api_requests", 1500, "method", "POST", "endpoint", "/users", "status", "200")
.counter("api_requests", 75, "method", "GET", "endpoint", "/users", "status", "404")
.counter("api_requests", 15, "method", "GET", "endpoint", "/users", "status", "500")
.timer("api_latency", new long[]{11, 14, 17, 22, 28, 35, 45}, "method", "GET", "endpoint", "/users")
.timer("api_latency", new long[]{9, 11, 13, 16, 19, 24}, "method", "GET", "endpoint", "/products")
.build();
}
/**
* Main method to generate the examples database.
*/
public static void main(String[] args) throws Exception {
Path examplesDb = Path.of("nb-apis/nb-mql-api/src/test/resources/testdata/examples.db");
System.out.println("Generating examples.db for documentation...");
generateExamples(examplesDb);
System.out.println("✓ Generated examples.db");
System.out.println("Location: " + examplesDb.toAbsolutePath());
}
}
@@ -0,0 +1,249 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.testdata;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer;
import io.nosqlbench.nb.api.engine.metrics.reporters.MetricInstanceFilter;
import io.nosqlbench.nb.api.engine.metrics.reporters.SqliteSnapshotReporter;
import io.nosqlbench.nb.api.engine.metrics.view.MetricsView;
import io.nosqlbench.nb.api.labels.NBLabels;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Simplified utility for creating test metrics databases with specific timestamps.
* Provides a fluent API for building predictable test data for documentation examples.
*
* Example:
* <pre>
* MetricsDataBuilder.create("examples.db")
* .atTime("2025-10-23T10:00:00Z")
* .counter("ops_total", 100, "activity", "read")
* .counter("ops_total", 150, "activity", "write")
* .atTime("2025-10-23T10:01:00Z")
* .counter("ops_total", 120, "activity", "read")
* .counter("ops_total", 180, "activity", "write")
* .build();
* </pre>
*/
public class MetricsDataBuilder {
private final Path outputPath;
private final List<Snapshot> snapshots = new ArrayList<>();
private Snapshot currentSnapshot;
private MetricsDataBuilder(Path outputPath) {
this.outputPath = outputPath;
}
/**
* Create a new metrics data builder for the specified database file.
*
* @param dbPath Path to the output database file
* @return New builder instance
*/
public static MetricsDataBuilder create(String dbPath) {
return new MetricsDataBuilder(Path.of(dbPath));
}
/**
* Create a new metrics data builder for the specified path.
*
* @param outputPath Path to the output database file
* @return New builder instance
*/
public static MetricsDataBuilder create(Path outputPath) {
return new MetricsDataBuilder(outputPath);
}
/**
* Start a new snapshot at the specified timestamp (ISO-8601 format).
*
* @param timestamp ISO-8601 timestamp (e.g., "2025-10-23T10:00:00Z")
* @return This builder for chaining
*/
public MetricsDataBuilder atTime(String timestamp) {
Instant instant = Instant.parse(timestamp);
return atTime(instant);
}
/**
* Start a new snapshot at the specified timestamp.
*
* @param timestamp Instant timestamp
* @return This builder for chaining
*/
public MetricsDataBuilder atTime(Instant timestamp) {
if (currentSnapshot != null) {
snapshots.add(currentSnapshot);
}
currentSnapshot = new Snapshot(timestamp);
return this;
}
/**
* Add a counter metric to the current snapshot.
*
* @param metricName Metric name (will have _total suffix added automatically)
* @param value Counter value
* @param labels Label key-value pairs (must be even number of strings)
* @return This builder for chaining
*/
public MetricsDataBuilder counter(String metricName, long value, String... labels) {
ensureCurrentSnapshot();
currentSnapshot.addCounter(metricName, value, parseLabels(labels));
return this;
}
/**
* Add a timer metric to the current snapshot.
*
* @param metricName Metric name
* @param latencies Array of latency values in milliseconds
* @param labels Label key-value pairs (must be even number of strings)
* @return This builder for chaining
*/
public MetricsDataBuilder timer(String metricName, long[] latencies, String... labels) {
ensureCurrentSnapshot();
currentSnapshot.addTimer(metricName, latencies, parseLabels(labels));
return this;
}
/**
* Build and write the database file.
*
* @throws Exception If database creation fails
*/
public void build() throws Exception {
if (currentSnapshot != null) {
snapshots.add(currentSnapshot);
}
if (snapshots.isEmpty()) {
throw new IllegalStateException("No snapshots defined. Use atTime() to add snapshots.");
}
NBComponent root = new NBBaseComponent(null, NBLabels.forKV("builder", "test"));
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
root, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
for (Snapshot snapshot : snapshots) {
snapshot.writeToReporter(reporter);
}
}
}
private void ensureCurrentSnapshot() {
if (currentSnapshot == null) {
throw new IllegalStateException("No current snapshot. Call atTime() first.");
}
}
private Map<String, String> parseLabels(String... labels) {
if (labels.length % 2 != 0) {
throw new IllegalArgumentException("Labels must be key-value pairs (even number of strings)");
}
Map<String, String> labelMap = new LinkedHashMap<>();
for (int i = 0; i < labels.length; i += 2) {
labelMap.put(labels[i], labels[i + 1]);
}
return labelMap;
}
/**
* Represents a single metrics snapshot at a specific point in time.
*/
private static class Snapshot {
private final Instant timestamp;
private final List<NBMetricCounter> counters = new ArrayList<>();
private final List<NBMetricTimer> timers = new ArrayList<>();
Snapshot(Instant timestamp) {
this.timestamp = timestamp;
}
void addCounter(String metricName, long value, Map<String, String> labels) {
Map<String, String> fullLabels = new LinkedHashMap<>(labels);
fullLabels.put("name", metricName);
NBMetricCounter counter = new NBMetricCounter(
NBLabels.forMap(fullLabels),
"counter",
MetricCategory.Core
);
counter.inc(value);
counters.add(counter);
}
void addTimer(String metricName, long[] latenciesMs, Map<String, String> labels) {
Map<String, String> fullLabels = new LinkedHashMap<>(labels);
fullLabels.put("name", metricName);
NBLabels nbLabels = NBLabels.forMap(fullLabels);
NBMetricTimer timer = new NBMetricTimer(
nbLabels,
new DeltaHdrHistogramReservoir(nbLabels, 3),
"timer",
MetricCategory.Core
);
for (long latencyMs : latenciesMs) {
timer.update(Duration.ofMillis(latencyMs));
}
timers.add(timer);
}
void writeToReporter(SqliteSnapshotReporter reporter) {
List<Object> allMetrics = new ArrayList<>();
allMetrics.addAll(counters);
allMetrics.addAll(timers);
@SuppressWarnings("unchecked")
List<io.nosqlbench.nb.api.engine.metrics.instruments.NBMetric> metrics =
(List<io.nosqlbench.nb.api.engine.metrics.instruments.NBMetric>) (List<?>) allMetrics;
MetricsView view = MetricsView.capture(metrics, 30000L);
// Override timestamp by capturing at specific time
// Note: MetricsView uses current time, but reporter will use it
reporter.onMetricsSnapshot(view);
}
}
/**
* Format an Instant as ISO-8601 for documentation.
*/
public static String formatTimestamp(Instant instant) {
return instant.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT);
}
}
@@ -0,0 +1,329 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.testdata;
import io.nosqlbench.nb.api.components.core.NBComponent;
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer;
import io.nosqlbench.nb.api.engine.metrics.reporters.SqliteSnapshotReporter;
import io.nosqlbench.nb.api.engine.metrics.reporters.MetricInstanceFilter;
import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
import io.nosqlbench.nb.api.engine.metrics.view.MetricsView;
import io.nosqlbench.nb.api.labels.NBLabels;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Generates test databases with various metric scenarios for testing query commands.
* Each generator method creates a database with specific characteristics documented
* in TEST_DATABASES.md.
*/
public class TestDatabaseGenerator {
/**
* Generate simple_counter.db
*
* A basic counter metric with two label dimensions over 5 minutes.
*
* Schema:
* - Metric: activity_ops_total (COUNTER)
* - Labels: activity={read,write}, host=server1
* - Time range: 5 minutes (10 snapshots @ 30s intervals)
* - Values: Linear growth (read: 0→100, write: 0→150)
*
* Purpose: Test instant queries, simple ranges, basic rate calculations
*/
public static void generateSimpleCounter(Path outputPath, NBComponent parent) throws Exception {
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
// Base timestamp: 2025-10-23 10:00:00 UTC
long baseTime = 1729681200000L;
long intervalMs = 30000L;
for (int i = 0; i < 10; i++) {
List<NBMetricCounter> counters = new ArrayList<>();
// Create read counter
NBMetricCounter readCounter = new NBMetricCounter(
NBLabels.forKV("name", "activity_ops", "activity", "read", "host", "server1"),
"ops",
MetricCategory.Core
);
readCounter.inc(i * 10); // 0, 10, 20, ..., 90
// Create write counter
NBMetricCounter writeCounter = new NBMetricCounter(
NBLabels.forKV("name", "activity_ops", "activity", "write", "host", "server1"),
"ops",
MetricCategory.Core
);
writeCounter.inc(i * 15); // 0, 15, 30, ..., 135
counters.add(readCounter);
counters.add(writeCounter);
// Create MetricsView and report snapshot
// Note: Timestamps will be auto-generated by the reporter
MetricsView view = MetricsView.capture(counters, intervalMs);
reporter.onMetricsSnapshot(view);
// Sleep to create distinct timestamps
Thread.sleep(100);
}
}
}
/**
* Generate multi_dimensional.db
*
* Complex metrics with 4 dimensional labels for testing grouping and filtering.
*
* Schema:
* - Metric: request_total (COUNTER)
* - Labels: activity={read,write}, host={server1,server2,server3},
* region={us-east,us-west}, env={prod,staging}
* - Combinations: 2 * 3 * 2 * 2 = 24 unique label sets
* - Time range: 2 minutes (4 snapshots @ 30s)
*
* Purpose: Test label filtering, aggregations with grouping, topk queries
*/
public static void generateMultiDimensional(Path outputPath, NBComponent parent) throws Exception {
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
long baseTime = 1729681200000L;
long intervalMs = 30000L;
String[] activities = {"read", "write"};
String[] hosts = {"server1", "server2", "server3"};
String[] regions = {"us-east", "us-west"};
String[] envs = {"prod", "staging"};
for (int snapshot = 0; snapshot < 4; snapshot++) {
List<NBMetricCounter> counters = new ArrayList<>();
int counter = 0;
for (String activity : activities) {
for (String host : hosts) {
for (String region : regions) {
for (String env : envs) {
NBMetricCounter metric = new NBMetricCounter(
NBLabels.forKV(
"name", "requests",
"activity", activity,
"host", host,
"region", region,
"env", env
),
"requests",
MetricCategory.Core
);
// Value varies by combination and snapshot
long value = (counter * 10) + (snapshot * 100);
metric.inc(value);
counters.add(metric);
counter++;
}
}
}
}
MetricsView view = MetricsView.capture(counters, intervalMs);
reporter.onMetricsSnapshot(view);
Thread.sleep(100);
}
}
}
/**
* Generate latency_timers.db
*
* Summary/timer metrics with quantiles for latency analysis.
*
* Schema:
* - Metric: operation_latency_ms (TIMER/SUMMARY)
* - Labels: operation={select,insert,update,delete}
* - Quantiles: p50, p75, p90, p95, p98, p99, p999
* - Statistics: count, sum, min, max, mean, stddev
* - Time range: 3 minutes (6 snapshots @ 30s)
*
* Purpose: Test quantile queries, statistical aggregations
*/
public static void generateLatencyTimers(Path outputPath, NBComponent parent) throws Exception {
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
long baseTime = 1729681200000L;
long intervalMs = 30000L;
String[] operations = {"select", "insert", "update", "delete"};
for (int snapshot = 0; snapshot < 6; snapshot++) {
List<NBMetricTimer> timers = new ArrayList<>();
for (int opIdx = 0; opIdx < operations.length; opIdx++) {
String operation = operations[opIdx];
NBLabels labels = NBLabels.forKV(
"name", "operation_latency",
"operation", operation
);
NBMetricTimer timer = new NBMetricTimer(
labels,
new DeltaHdrHistogramReservoir(labels, 3),
"latency",
MetricCategory.Core
);
// Vary latencies by operation type and over time
long baseLatency = (opIdx + 1) * 10_000_000L; // 10ms, 20ms, 30ms, 40ms in nanoseconds
long drift = snapshot * 2_000_000L; // 2ms increase per snapshot
// Record latencies with some variation
int recordCount = 100 + (snapshot * 20);
for (int i = 0; i < recordCount; i++) {
// Create a distribution of latencies
long latency;
if (i < recordCount * 0.5) {
latency = baseLatency + drift; // p50
} else if (i < recordCount * 0.75) {
latency = (long) ((baseLatency + drift) * 1.5); // p75
} else if (i < recordCount * 0.95) {
latency = (long) ((baseLatency + drift) * 2.0); // p95
} else {
latency = (long) ((baseLatency + drift) * 3.0); // p99
}
timer.update(Duration.ofNanos(latency));
}
timers.add(timer);
}
long timestamp = baseTime + (snapshot * intervalMs);
MetricsView view = MetricsView.capture(timers, intervalMs);
reporter.onMetricsSnapshot(view);
Thread.sleep(100);
}
}
}
/**
* Generate rate_calculations.db
*
* Counter metrics with predictable growth patterns for testing rate() and increase().
*
* Schema:
* - Metrics: counter_total (COUNTER)
* - Labels: pattern={linear,exponential,step}
* - Time range: 1 hour (120 snapshots @ 30s)
* - Patterns:
* - linear: grows by 10 per snapshot (constant rate)
* - exponential: grows by 1.01x per snapshot
* - step: 100 for 10 snapshots, then jump to 200 for 10, etc.
*
* Purpose: Test rate() and increase() with known expected results
*/
public static void generateRateCalculations(Path outputPath, NBComponent parent) throws Exception {
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
long baseTime = 1729681200000L;
long intervalMs = 30000L;
double exponentialValue = 100.0;
for (int i = 0; i < 120; i++) {
List<NBMetricCounter> counters = new ArrayList<>();
// Linear counter: constant rate of 10 per 30s = 0.333 ops/sec
NBMetricCounter linearCounter = new NBMetricCounter(
NBLabels.forKV("name", "patterns", "pattern", "linear"),
"counter",
MetricCategory.Core
);
linearCounter.inc(i * 10);
counters.add(linearCounter);
// Exponential counter: exponential growth
exponentialValue *= 1.01; // 1% growth per snapshot
NBMetricCounter expCounter = new NBMetricCounter(
NBLabels.forKV("name", "patterns", "pattern", "exponential"),
"counter",
MetricCategory.Core
);
expCounter.inc((long) exponentialValue);
counters.add(expCounter);
// Step counter: steps every 10 snapshots
long stepValue = ((i / 10) + 1) * 100;
NBMetricCounter stepCounter = new NBMetricCounter(
NBLabels.forKV("name", "patterns", "pattern", "step"),
"counter",
MetricCategory.Core
);
stepCounter.inc(stepValue);
counters.add(stepCounter);
MetricsView view = MetricsView.capture(counters, intervalMs);
reporter.onMetricsSnapshot(view);
Thread.sleep(50); // Shorter sleep for many snapshots
}
}
}
/**
* Main method to generate all test databases.
* Run this to regenerate test data after schema changes.
*/
public static void main(String[] args) throws Exception {
Path testDataDir = Path.of("nb-apis/nb-mql-api/src/test/resources/testdata");
// Create a minimal NBComponent for the reporter
NBComponent root = new NBBaseComponent(null, NBLabels.forKV("generator", "testdata"));
System.out.println("Generating test databases...");
generateSimpleCounter(testDataDir.resolve("simple_counter.db"), root);
System.out.println("✓ Generated simple_counter.db");
generateMultiDimensional(testDataDir.resolve("multi_dimensional.db"), root);
System.out.println("✓ Generated multi_dimensional.db");
generateLatencyTimers(testDataDir.resolve("latency_timers.db"), root);
System.out.println("✓ Generated latency_timers.db");
generateRateCalculations(testDataDir.resolve("rate_calculations.db"), root);
System.out.println("✓ Generated rate_calculations.db");
System.out.println("\nAll test databases generated successfully!");
System.out.println("Location: " + testDataDir.toAbsolutePath());
}
}
@@ -0,0 +1,128 @@
/*
* Copyright (c) nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.nb.mql.testdata;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Utility for loading test databases from src/test/resources/testdata/.
* Provides easy access to pre-generated test databases for unit and integration tests.
*/
public class TestDatabaseLoader {
private static final String TEST_DATA_DIR = "testdata";
/**
* Available test databases with their purposes.
*/
public enum TestDatabase {
SIMPLE_COUNTER("simple_counter.db", "Basic counters for instant/rate queries"),
MULTI_DIMENSIONAL("multi_dimensional.db", "Complex 4D labels for aggregations"),
LATENCY_TIMERS("latency_timers.db", "Timer metrics with quantiles"),
RATE_CALCULATIONS("rate_calculations.db", "Predictable growth patterns for rate testing");
private final String filename;
private final String description;
TestDatabase(String filename, String description) {
this.filename = filename;
this.description = description;
}
public String getFilename() {
return filename;
}
public String getDescription() {
return description;
}
}
/**
* Get the path to a test database by enum.
*
* @param database The test database enum
* @return Path to the database file
* @throws IOException If the database file cannot be found
*/
public static Path getDatabase(TestDatabase database) throws IOException {
return getDatabase(database.getFilename());
}
/**
* Get the path to a test database by filename.
*
* @param filename Name of the database file (e.g., "simple_counter.db")
* @return Path to the database file
* @throws IOException If the database file cannot be found
*/
public static Path getDatabase(String filename) throws IOException {
String resourcePath = TEST_DATA_DIR + "/" + filename;
URL resource = TestDatabaseLoader.class.getClassLoader().getResource(resourcePath);
if (resource == null) {
throw new IOException(
"Test database not found: " + resourcePath + "\n" +
"Make sure test databases have been generated by running TestDatabaseGenerator.main()"
);
}
try {
return Paths.get(resource.toURI());
} catch (URISyntaxException e) {
throw new IOException("Invalid URI for test database: " + resourcePath, e);
}
}
/**
* Check if a test database exists.
*
* @param database The test database enum
* @return true if the database file exists
*/
public static boolean exists(TestDatabase database) {
try {
getDatabase(database);
return true;
} catch (IOException e) {
return false;
}
}
/**
* Get a descriptive message about available test databases.
*
* @return Multi-line string describing all test databases
*/
public static String getAvailableDatabases() {
StringBuilder sb = new StringBuilder("Available test databases:\n");
for (TestDatabase db : TestDatabase.values()) {
sb.append(String.format(" - %s: %s\n",
db.getFilename(),
db.getDescription()));
}
return sb.toString();
}
private TestDatabaseLoader() {
// Utility class - prevent instantiation
}
}
@@ -0,0 +1,310 @@
# Test Database Documentation
This directory contains pre-generated SQLite metrics databases for testing the MetricsQL query engine. Each database represents a specific scenario with known data patterns.
## Overview
All databases use the NoSQLBench OpenMetrics-aligned schema created by `SqliteSnapshotReporter`. They contain realistic metric data generated using actual metric instruments (`NBMetricCounter`, `NBMetricTimer`).
---
## simple_counter.db
### Purpose
Test basic counter queries, instant values, simple rate calculations, and time-series ranges.
### Schema
- **Metric Family**: `activity_ops`
- **Sample**: `activity_ops_total` (COUNTER)
- **Type**: COUNTER
### Label Dimensions
- `activity`: {read, write}
- `host`: {server1}
### Data Characteristics
- **Snapshots**: 10 (taken ~100ms apart during generation)
- **Label Combinations**: 2 (read, write)
- **Total Samples**: 20
- **Growth Pattern**: Linear increments
- `activity=read`: increments by 10 per snapshot (0, 10, 20, ..., 90)
- `activity=write`: increments by 15 per snapshot (0, 15, 30, ..., 135)
### Expected Query Results
**Instant Query (latest value):**
```sql
SELECT sv.value, lv.value as activity
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
JOIN label_key lk ON lsm.label_key_id = lk.id
JOIN label_value lv ON lsm.label_value_id = lv.id
WHERE sn.sample = 'activity_ops_total'
AND lk.name = 'activity'
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
ORDER BY activity;
```
Expected: `{activity=read: 90, activity=write: 135}`
**Count Total Samples:**
```sql
SELECT COUNT(*) FROM sample_value;
```
Expected: `20`
**Rate Calculation (approximate):**
For `activity=read`, the rate should be approximately **10 ops per 100ms** = **100 ops/sec** (simplified; actual timing varies slightly)
### Use Cases
- Testing instant value queries
- Basic label filtering
- Simple rate calculations
- Time-series data retrieval
---
## multi_dimensional.db
### Purpose
Test complex label filtering, multi-dimensional aggregations, grouping operations, and topk queries.
### Schema
- **Metric Family**: `requests`
- **Sample**: `request_total` (COUNTER)
- **Type**: COUNTER
### Label Dimensions
- `activity`: {read, write}
- `host`: {server1, server2, server3}
- `region`: {us-east, us-west}
- `env`: {prod, staging}
### Data Characteristics
- **Snapshots**: 4
- **Label Combinations**: 24 (2 × 3 × 2 × 2)
- **Total Samples**: 96
- **Value Pattern**: `(combo_index * 10) + (snapshot_index * 100)`
- First snapshot: values 0, 10, 20, ..., 230
- Second snapshot: values 100, 110, 120, ..., 330
- And so on...
### Expected Query Results
**Count by Environment:**
```sql
SELECT lv.value as env, COUNT(*) as sample_count
FROM sample_value sv
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
JOIN label_key lk ON lsm.label_key_id = lk.id
JOIN label_value lv ON lsm.label_value_id = lv.id
WHERE lk.name = 'env'
GROUP BY env;
```
Expected: `{prod: 48, staging: 48}`
**Unique Label Sets:**
```sql
SELECT COUNT(DISTINCT label_set_id) FROM sample_value;
```
Expected: `24`
### Use Cases
- Multi-dimensional filtering (e.g., `activity=write AND region=us-east AND env=prod`)
- Aggregations with grouping (e.g., `sum by (region, env)`)
- Top-K queries (e.g., "top 5 hosts by request count")
- Complex label pattern matching
---
## latency_timers.db
### Purpose
Test summary/timer metrics with quantiles, statistical aggregations, and rate statistics.
### Schema
- **Metric Family**: `operation_latency`
- **Sample**: `operation_latency_ms` (TIMER/SUMMARY)
- **Type**: TIMER
### Label Dimensions
- `operation`: {select, insert, update, delete}
### Data Characteristics
- **Snapshots**: 6
- **Operations**: 4
- **Total Sample Values**: 24
- **Quantiles**: p50, p75, p90, p95, p98, p99, p999 (stored in `sample_quantile` table)
- **Statistics**: count, sum, min, max, mean, stddev (in `sample_value`)
- **Rate Statistics**: mean, m1, m5, m15 (in `sample_rate` table)
### Latency Patterns
Base latencies by operation (in nanoseconds):
- `select`: 10ms base, increases by 2ms per snapshot
- `insert`: 20ms base, increases by 2ms per snapshot
- `update`: 30ms base, increases by 2ms per snapshot
- `delete`: 40ms base, increases by 2ms per snapshot
Distribution within each snapshot:
- 50% of samples at base latency (p50)
- 25% at 1.5× base (p75)
- 20% at 2.0× base (p95)
- 5% at 3.0× base (p99)
### Expected Query Results
**Get p95 latency for select operations:**
```sql
SELECT sq.quantile_value
FROM sample_quantile sq
JOIN sample_value sv ON sq.sample_value_id = sv.id
JOIN sample_name sn ON sv.sample_name_id = sn.id
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
JOIN label_key lk ON lsm.label_key_id = lk.id
JOIN label_value lv ON lsm.label_value_id = lv.id
WHERE sn.sample = 'operation_latency_ms'
AND lk.name = 'operation'
AND lv.value = 'select'
AND sq.quantile = 0.95
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
LIMIT 1;
```
Expected: Approximately `22,000,000` nanoseconds (22ms) for latest snapshot
**Count quantile entries:**
```sql
SELECT COUNT(*) FROM sample_quantile;
```
Expected: `168` (24 samples × 7 quantiles each)
### Use Cases
- Quantile extraction (p50, p95, p99)
- Statistical aggregations (mean, stddev, min, max)
- Latency trend analysis over time
- Rate statistics queries (m1, m5, m15)
---
## rate_calculations.db
### Purpose
Test rate() and increase() commands with predictable, mathematically known patterns over extended time periods.
### Schema
- **Metric Family**: `patterns`
- **Sample**: `counter_total` (COUNTER)
- **Type**: COUNTER
### Label Dimensions
- `pattern`: {linear, exponential, step}
### Data Characteristics
- **Snapshots**: 120 (simulating 1 hour at 30s intervals)
- **Patterns**: 3
- **Total Samples**: 360
- **Interval Between Snapshots**: ~50ms (during generation)
### Growth Patterns
#### Linear Pattern (`pattern=linear`)
- Growth: **+10 per snapshot**
- Values: 0, 10, 20, 30, ..., 1190
- Expected rate: **10 ops / 50ms** = **200 ops/sec** (constant)
- Total increase over all snapshots: **1190**
#### Exponential Pattern (`pattern=exponential`)
- Growth: **×1.01 per snapshot** (1% compound growth)
- Starting value: 100
- Values: 100, 101, 102.01, 103.03, ...
- Expected rate: Increases over time
- Formula: `value[i] = 100 × (1.01)^i`
#### Step Pattern (`pattern=step`)
- Growth: **Steps every 10 snapshots**
- Values: 100 (snapshots 0-9), 200 (10-19), 300 (20-29), ..., 1200 (110-119)
- Expected rate: **0 within step, jump at boundaries**
- Total steps: 12
### Expected Query Results
**Linear pattern increase over all snapshots:**
```sql
SELECT MAX(sv.value) - MIN(sv.value) as total_increase
FROM sample_value sv
JOIN sample_name sn ON sv.sample_name_id = sn.id
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
JOIN label_key lk ON lsm.label_key_id = lk.id
JOIN label_value lv ON lsm.label_value_id = lv.id
WHERE sn.sample = 'counter_total'
AND lk.name = 'pattern'
AND lv.value = 'linear';
```
Expected: `1190`
**Count snapshots per pattern:**
```sql
SELECT lv.value as pattern, COUNT(*) as snapshots
FROM sample_value sv
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
JOIN label_key lk ON lsm.label_key_id = lk.id
JOIN label_value lv ON lsm.label_value_id = lv.id
WHERE lk.name = 'pattern'
GROUP BY pattern;
```
Expected: `{linear: 120, exponential: 120, step: 120}`
### Use Cases
- Testing rate() with constant rate (linear)
- Testing rate() with changing rate (exponential)
- Testing rate() with discontinuous data (step)
- Testing increase() over various time windows
- Validating rate calculation algorithms with known results
---
## Schema Reference
All databases share this schema (created by `SqliteSnapshotReporter`):
```
metric_family (id, name, help, unit, type)
└─ sample_name (id, metric_family_id, sample)
└─ sample_value (id, sample_name_id, label_set_id, timestamp_ms, value, count, sum, min, max, mean, stddev)
├─ sample_quantile (sample_value_id, quantile, quantile_value)
├─ sample_rate (sample_value_id, rate_type, rate_value)
└─ sample_histogram (sample_value_id, start_seconds, interval_seconds, max_value, histogram_base64)
label_key (id, name)
label_value (id, value)
label_set (id, hash)
label_set_membership (label_set_id, label_key_id, label_value_id)
```
## Regenerating Test Databases
To regenerate all test databases (e.g., after schema changes):
```bash
mvn exec:java -pl nb-apis/nb-mql-api \
-Dexec.mainClass="io.nosqlbench.nb.mql.testdata.TestDatabaseGenerator" \
-Dexec.classpathScope=test
```
The generator will overwrite existing databases in `src/test/resources/testdata/`.
## Using in Tests
Use `TestDatabaseLoader` utility to access these databases in your tests:
```java
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
Path dbPath = TestDatabaseLoader.getDatabase("simple_counter.db");
Connection conn = MetricsDatabaseReader.connect(dbPath);
// ... run queries
```
---
*Generated: 2025-10-23*
*Schema Version: OpenMetrics-aligned (SqliteSnapshotReporter)*
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
View File
@@ -35,6 +35,7 @@
<module>adapters-api</module>
<module>expr-api</module>
<module>nb-api</module>
<module>nb-mql-api</module>
<module>nb-apis-all</module>
</modules>
+6
View File
@@ -55,6 +55,12 @@
<artifactId>nb-engine-extensions</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>nb-mql-api</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
<build>
<!-- <plugins>-->
@@ -182,6 +182,11 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
.activate();
ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig); // THIS should be the first time log4j2 is invoked!
// Add shutdown handler to timestamp symlinks when session ends
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
NBCLI.loggerConfig.timestampSymlinksOnShutdown();
}, "symlink-timestamp-shutdown"));
NBCLI.logger = LogManager.getLogger("NBCLI"); // TODO: Detect if the logger config was already initialized (error)
NBCLI.loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
if (NBCLI.logger.isInfoEnabled())
@@ -1,4 +1,4 @@
# Metrics Reporting Guide
contin# Metrics Reporting Guide
## What You Can Configure
@@ -485,4 +485,42 @@ public class NBLoggerConfig extends ConfigurationFactory {
}
/**
* Update modification time on symlinks to reflect shutdown time.
* Called by shutdown hook to touch symlinks so their mtime shows when the session ended.
* Updates symlinks for: scenario.log, status.log, runtime.log, verify.log, metrics.db, heartbeat.log
*/
public void timestampSymlinksOnShutdown() {
if (loggerDir == null) {
return;
}
// List of symlinks to update
String[] symlinks = {
"scenario.log",
"status.log",
"runtime.log",
"verify.log",
"metrics.db",
"heartbeat.log"
};
for (String linkName : symlinks) {
try {
Path linkPath = loggerDir.resolve(linkName);
if (!Files.exists(linkPath, java.nio.file.LinkOption.NOFOLLOW_LINKS)) {
continue; // Symlink doesn't exist, skip
}
// Update the modification time of the symlink itself (not the target file)
Files.setLastModifiedTime(linkPath,
java.nio.file.attribute.FileTime.from(java.time.Instant.now()));
} catch (IOException | UnsupportedOperationException e) {
// Silently ignore - symlinks may not be supported or may have issues during shutdown
}
}
}
}