diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 833486e7d..74ca378bd 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -181,6 +181,11 @@ ascii-data 1.4.0 + + org.xerial + sqlite-jdbc + 3.46.0.0 + org.lz4 diff --git a/nb-apis/nb-api/pom.xml b/nb-apis/nb-api/pom.xml index bf505dcf8..1f0bc3426 100644 --- a/nb-apis/nb-api/pom.xml +++ b/nb-apis/nb-api/pom.xml @@ -159,7 +159,6 @@ org.xerial sqlite-jdbc - 3.46.0.0 diff --git a/nb-apis/nb-apis-all/pom.xml b/nb-apis/nb-apis-all/pom.xml index 8527e1d0f..ab53ae743 100644 --- a/nb-apis/nb-apis-all/pom.xml +++ b/nb-apis/nb-apis-all/pom.xml @@ -47,5 +47,10 @@ adapters-api ${revision} + + io.nosqlbench + nb-mql-api + ${revision} + diff --git a/nb-apis/nb-mql-api/pom.xml b/nb-apis/nb-mql-api/pom.xml new file mode 100644 index 000000000..7aa0e4c57 --- /dev/null +++ b/nb-apis/nb-mql-api/pom.xml @@ -0,0 +1,139 @@ + + + + 4.0.0 + + + mvn-defaults + io.nosqlbench + ${revision} + ../../mvn-defaults + + + nb-mql-api + jar + ${project.artifactId} + + MetricsQL-inspired query engine for NoSQLBench SQLite metrics databases. + Provides read-only query commands for analyzing metrics data with support + for rates, aggregations, quantiles, and time-series analysis. + + + + + + io.nosqlbench + nb-api + ${revision} + + + + + org.xerial + sqlite-jdbc + + + + + info.picocli + picocli + + + + + com.google.code.gson + gson + + + + + org.apache.commons + commons-csv + + + + + org.slf4j + slf4j-api + 2.0.16 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.25.2 + + + org.apache.logging.log4j + log4j-core + 2.25.2 + + + org.apache.logging.log4j + log4j-api + 2.25.2 + + + + + + + + + src/test/resources + false + + **/*.db + + + + src/test/resources + true + + **/*.db + + + + + + + + maven-assembly-plugin + + + + io.nosqlbench.nb.mql.cli.MetricsQLCLI + + + + jar-with-dependencies + + nb-metricsql + + + + make-assembly + package + + single + + + + + + + + diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/MQLApp.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/MQLApp.java new file mode 100644 index 000000000..2723c12e0 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/MQLApp.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql; + +import io.nosqlbench.nb.annotations.Service; +import io.nosqlbench.nb.api.apps.BundledApp; +import io.nosqlbench.nb.mql.cli.MetricsQLCLI; +import picocli.CommandLine; + +/** + * MetricsQL Query Engine as a NoSQLBench BundledApp. + * Allows querying SQLite metrics databases using MetricsQL-inspired syntax. + * + * Usage: + * nb5 mql instant --metric ops_total + * nb5 mql range --metric ops_total --window 5m + * + * This integrates the MetricsQL CLI as a bundled application within NoSQLBench, + * making it immediately available without needing a separate tool. + */ +@Service(value = BundledApp.class, selector = "mql") +public class MQLApp implements BundledApp { + + @Override + public int applyAsInt(String[] args) { + // Delegate to the picocli-based MetricsQLCLI + // This allows all the command routing and validation to work seamlessly + return new CommandLine(new MetricsQLCLI()).execute(args); + } + + /** + * Standalone main method for direct execution (optional). + * Can also be run via: java -cp ... io.nosqlbench.nb.mql.MQLApp instant --metric ops + */ + public static void main(String[] args) { + MQLApp app = new MQLApp(); + int exitCode = app.applyAsInt(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/AbstractQueryCommandCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/AbstractQueryCommandCLI.java new file mode 100644 index 000000000..21a278ab8 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/AbstractQueryCommandCLI.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.format.OutputFormat; +import io.nosqlbench.nb.mql.format.ResultFormatter; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * Abstract base class for all query command CLIs. + * Provides common functionality for database connection, formatting, and error handling. + * + * Subclasses must implement: + * - createCommand() - Create the specific MetricsQueryCommand instance + * - buildParams() - Build the parameter map for the command + */ +public abstract class AbstractQueryCommandCLI implements Callable { + + @Option( + names = {"-d", "--db"}, + description = "Path to metrics database (default: logs/metrics.db)", + defaultValue = "logs/metrics.db" + ) + protected Path databasePath; + + @Option( + names = {"-m", "--metric"}, + description = "Metric name to query", + required = true + ) + protected String metric; + + @Option( + names = {"-l", "--labels"}, + description = "Label filters as key=value,key=value", + split = "," + ) + protected Map labels = new LinkedHashMap<>(); + + @Option( + names = {"-f", "--format"}, + description = "Output format: table, json, jsonl, csv, tsv, markdown", + defaultValue = "table" + ) + protected OutputFormat format = OutputFormat.TABLE; + + @Option( + names = {"--explain"}, + description = "Show the SQL query instead of executing" + ) + protected boolean explain; + + @Override + public Integer call() throws Exception { + try (Connection conn = MetricsDatabaseReader.connect(databasePath)) { + MetricsQueryCommand command = createCommand(); + Map params = buildParams(); + + QueryResult result = command.execute(conn, params); + + if (explain) { + System.out.println("SQL Query:"); + System.out.println(result.executedSQL()); + return 0; + } + + ResultFormatter formatter = format.createFormatter(); + System.out.println(formatter.format(result)); + + return 0; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + if (System.getenv("DEBUG") != null) { + e.printStackTrace(); + } + return 1; + } + } + + /** + * Create the specific command instance for this CLI. + * + * @return MetricsQueryCommand instance + */ + protected abstract MetricsQueryCommand createCommand(); + + /** + * Build the parameter map for the command. + * Subclasses override to add command-specific parameters. + * + * @return Parameter map + */ + protected Map buildParams() { + Map params = new LinkedHashMap<>(); + params.put("metric", metric); + if (!labels.isEmpty()) { + params.put("labels", labels); + } + return params; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/AggregateCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/AggregateCLI.java new file mode 100644 index 000000000..7b3d39922 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/AggregateCLI.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.AggregateCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * CLI for aggregate query command. + * Performs aggregations (sum, avg, max, min, count) with optional grouping. + */ +@Command( + name = "agg", + description = "Perform aggregations (sum, avg, max, min, count) on metrics", + mixinStandardHelpOptions = true +) +public class AggregateCLI extends AbstractQueryCommandCLI { + + @Option( + names = {"--func", "--function"}, + description = "Aggregate function: sum, avg, max, min, count", + required = true + ) + private String function; + + @Option( + names = {"--by"}, + description = "Group by labels (comma-separated, e.g., activity,host)", + split = "," + ) + private List groupBy; + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h) - optional, defaults to latest snapshot" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window" + ) + private String last; + + @Override + protected MetricsQueryCommand createCommand() { + return new AggregateCommand(); + } + + @Override + protected Map buildParams() { + Map params = super.buildParams(); + + params.put("function", function); + + if (groupBy != null && !groupBy.isEmpty()) { + params.put("by", groupBy); + } + + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } + + return params; + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new AggregateCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/IncreaseCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/IncreaseCLI.java new file mode 100644 index 000000000..4fc654464 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/IncreaseCLI.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.IncreaseCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.Map; + +/** + * CLI for increase query command. + * Calculates total increase in counter value over a time window. + */ +@Command( + name = "increase", + description = "Calculate total increase in counter over time window", + mixinStandardHelpOptions = true +) +public class IncreaseCLI extends AbstractQueryCommandCLI { + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h, 30s)" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window (e.g., last 5m)" + ) + private String last; + + @Override + protected MetricsQueryCommand createCommand() { + return new IncreaseCommand(); + } + + @Override + protected Map buildParams() { + Map params = super.buildParams(); + + // Add time window parameter + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } + + return params; + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new IncreaseCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/InstantCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/InstantCLI.java new file mode 100644 index 000000000..0acfb283a --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/InstantCLI.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.InstantCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +import java.util.Map; + +/** + * CLI for instant query command. + * Can be run standalone: java -cp ... io.nosqlbench.nb.mql.cli.InstantCLI --metric ops_total + */ +@Command( + name = "instant", + description = "Get the most recent value for a metric", + mixinStandardHelpOptions = true +) +public class InstantCLI extends AbstractQueryCommandCLI { + + @Override + protected MetricsQueryCommand createCommand() { + return new InstantCommand(); + } + + @Override + protected Map buildParams() { + // Use base class params (metric and labels) + return super.buildParams(); + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new InstantCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/MetricsQLCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/MetricsQLCLI.java new file mode 100644 index 000000000..466344f09 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/MetricsQLCLI.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import picocli.CommandLine; +import picocli.CommandLine.Command; + +/** + * Main entry point for the MetricsQL CLI tool. + * Routes to subcommands for different query types. + * + * Usage: + * nb-metricsql instant --metric ops_total + * nb-metricsql range --metric ops_total --window 5m + * nb-metricsql rate --metric ops_total --window 5m + * ... + */ +@Command( + name = "nb-metricsql", + description = "MetricsQL query engine for NoSQLBench SQLite metrics databases", + mixinStandardHelpOptions = true, + version = "1.0.0", + subcommands = { + SessionCLI.class, + SummaryCLI.class, + InstantCLI.class, + RangeCLI.class, + RateCLI.class, + IncreaseCLI.class, + QuantileCLI.class, + AggregateCLI.class, + RatioCLI.class, + TopKCLI.class, + SqlCLI.class + // All commands implemented except: + // BinaryOpCLI.class (optional - advanced arithmetic) + } +) +public class MetricsQLCLI implements Runnable { + + @Override + public void run() { + // If no subcommand specified, show help + CommandLine.usage(this, System.out); + } + + /** + * Main entry point for the standalone JAR. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new MetricsQLCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/QuantileCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/QuantileCLI.java new file mode 100644 index 000000000..9576a8e10 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/QuantileCLI.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.QuantileCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.Map; + +/** + * CLI for quantile query command. + * Extracts percentile values from timer/summary metrics. + */ +@Command( + name = "quantile", + description = "Extract percentile values from timer/summary metrics", + mixinStandardHelpOptions = true +) +public class QuantileCLI extends AbstractQueryCommandCLI { + + @Option( + names = {"-p", "--percentile", "--quantile"}, + description = "Quantile value (0.0-1.0, e.g., 0.95 for p95)", + required = true + ) + private double quantile; + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h) - optional, defaults to all data" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window (e.g., last 5m)" + ) + private String last; + + @Override + protected MetricsQueryCommand createCommand() { + return new QuantileCommand(); + } + + @Override + protected Map buildParams() { + Map params = super.buildParams(); + + params.put("quantile", quantile); + + // Add time window if specified + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } + + return params; + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new QuantileCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RangeCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RangeCLI.java new file mode 100644 index 000000000..eb365a7ba --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RangeCLI.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.RangeCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.Map; + +/** + * CLI for range query command. + * Can be run standalone: java -cp ... io.nosqlbench.nb.mql.cli.RangeCLI --metric ops_total --window 5m + */ +@Command( + name = "range", + description = "Retrieve time-series data over a time window", + mixinStandardHelpOptions = true +) +public class RangeCLI extends AbstractQueryCommandCLI { + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h, 30s)" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window (e.g., last 5m)" + ) + private String last; + + @Option( + names = {"--start"}, + description = "Start timestamp in epoch milliseconds" + ) + private Long start; + + @Option( + names = {"--end"}, + description = "End timestamp in epoch milliseconds" + ) + private Long end; + + @Override + protected MetricsQueryCommand createCommand() { + return new RangeCommand(); + } + + @Override + protected Map buildParams() { + Map params = super.buildParams(); + + // Add time range parameters + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } else if (start != null && end != null) { + params.put("start", start); + params.put("end", end); + } + + return params; + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new RangeCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RateCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RateCLI.java new file mode 100644 index 000000000..049b01545 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RateCLI.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.RateCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.Map; + +/** + * CLI for rate query command. + * Calculates per-second rate of change for counter metrics. + */ +@Command( + name = "rate", + description = "Calculate per-second rate of change for counters", + mixinStandardHelpOptions = true +) +public class RateCLI extends AbstractQueryCommandCLI { + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h, 30s)" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window (e.g., last 5m)" + ) + private String last; + + @Override + protected MetricsQueryCommand createCommand() { + return new RateCommand(); + } + + @Override + protected Map buildParams() { + Map params = super.buildParams(); + + // Add time window parameter + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } + + return params; + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new RateCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RatioCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RatioCLI.java new file mode 100644 index 000000000..b93733451 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/RatioCLI.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.RatioCommand; +import io.nosqlbench.nb.mql.format.OutputFormat; +import io.nosqlbench.nb.mql.format.ResultFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * CLI for ratio query command. + * Calculates ratio between two metrics (useful for error rates). + */ +@Command( + name = "ratio", + description = "Calculate ratio between two metrics (numerator / denominator)", + mixinStandardHelpOptions = true +) +public class RatioCLI implements Callable { + + @Option( + names = {"-d", "--db"}, + description = "Path to metrics database (default: logs/metrics.db)", + defaultValue = "logs/metrics.db" + ) + private Path databasePath; + + @Option( + names = {"-n", "--numerator"}, + description = "Numerator metric name", + required = true + ) + private String numerator; + + @Option( + names = {"--denominator"}, + description = "Denominator metric name", + required = true + ) + private String denominator; + + @Option( + names = {"-l", "--labels"}, + description = "Label filters (applied to both metrics)", + split = "," + ) + private Map labels = new LinkedHashMap<>(); + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h)" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window" + ) + private String last; + + @Option( + names = {"-f", "--format"}, + description = "Output format: table, json, jsonl, csv, tsv, markdown", + defaultValue = "table" + ) + private OutputFormat format = OutputFormat.TABLE; + + @Option( + names = {"--explain"}, + description = "Show the SQL query instead of executing" + ) + private boolean explain; + + @Override + public Integer call() throws Exception { + try (Connection conn = MetricsDatabaseReader.connect(databasePath)) { + RatioCommand command = new RatioCommand(); + + Map params = new LinkedHashMap<>(); + params.put("numerator", numerator); + params.put("denominator", denominator); + + if (!labels.isEmpty()) { + params.put("labels", labels); + } + + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } + + QueryResult result = command.execute(conn, params); + + if (explain) { + System.out.println("SQL Query:"); + System.out.println(result.executedSQL()); + return 0; + } + + ResultFormatter formatter = format.createFormatter(); + System.out.println(formatter.format(result)); + + return 0; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + if (System.getenv("DEBUG") != null) { + e.printStackTrace(); + } + return 1; + } + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new RatioCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SessionCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SessionCLI.java new file mode 100644 index 000000000..4e48af093 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SessionCLI.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.SessionCommand; +import io.nosqlbench.nb.mql.format.OutputFormat; +import io.nosqlbench.nb.mql.format.ResultFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * CLI for session command. + * Displays session time information. + */ +@Command( + name = "session", + description = "Display session time information and snapshot cadence", + mixinStandardHelpOptions = true +) +public class SessionCLI implements Callable { + + @Option( + names = {"-d", "--db"}, + description = "Path to metrics database (default: logs/metrics.db)", + defaultValue = "logs/metrics.db" + ) + private Path databasePath; + + @Option( + names = {"-f", "--format"}, + description = "Output format: table, json, jsonl, csv, tsv, markdown", + defaultValue = "table" + ) + private OutputFormat format = OutputFormat.TABLE; + + @Override + public Integer call() throws Exception { + try (Connection conn = MetricsDatabaseReader.connect(databasePath)) { + SessionCommand command = new SessionCommand(); + QueryResult result = command.execute(conn, Map.of()); + + ResultFormatter formatter = format.createFormatter(); + System.out.println(formatter.format(result)); + + return 0; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + if (System.getenv("DEBUG") != null) { + e.printStackTrace(); + } + return 1; + } + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new SessionCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SqlCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SqlCLI.java new file mode 100644 index 000000000..c97cae1c9 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SqlCLI.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.SqlCommand; +import io.nosqlbench.nb.mql.format.OutputFormat; +import io.nosqlbench.nb.mql.format.ResultFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * CLI for raw SQL query execution. + * Allows direct SQLite queries for advanced users. + */ +@Command( + name = "sql", + description = "Execute raw SQLite queries (read-only)", + mixinStandardHelpOptions = true +) +public class SqlCLI implements Callable { + + @Option( + names = {"-d", "--db"}, + description = "Path to metrics database (default: logs/metrics.db)", + defaultValue = "logs/metrics.db" + ) + private Path databasePath; + + @Option( + names = {"-q", "--query"}, + description = "SQL query to execute (SELECT, WITH, or PRAGMA only)", + required = true + ) + private String query; + + @Option( + names = {"-t", "--timeout"}, + description = "Query timeout in seconds (default: 30)", + defaultValue = "30" + ) + private int timeout; + + @Option( + names = {"-f", "--format"}, + description = "Output format: table, json, jsonl, csv, tsv, markdown", + defaultValue = "table" + ) + private OutputFormat format = OutputFormat.TABLE; + + @Override + public Integer call() throws Exception { + try (Connection conn = MetricsDatabaseReader.connect(databasePath)) { + SqlCommand command = new SqlCommand(); + + Map params = new LinkedHashMap<>(); + params.put("query", query); + params.put("timeout", timeout); + + QueryResult result = command.execute(conn, params); + + ResultFormatter formatter = format.createFormatter(); + System.out.println(formatter.format(result)); + + return 0; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + if (System.getenv("DEBUG") != null) { + e.printStackTrace(); + } + return 1; + } + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new SqlCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SummaryCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SummaryCLI.java new file mode 100644 index 000000000..655f9fdaa --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/SummaryCLI.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.SummaryCommand; +import io.nosqlbench.nb.mql.format.OutputFormat; +import io.nosqlbench.nb.mql.format.ResultFormatter; +import io.nosqlbench.nb.mql.format.SummaryMarkdownFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * CLI for summary command. + * Displays database overview and metrics inventory. + */ +@Command( + name = "summary", + description = "Display overview of metrics in the database", + mixinStandardHelpOptions = true +) +public class SummaryCLI implements Callable { + + @Option( + names = {"-d", "--db"}, + description = "Path to metrics database (default: logs/metrics.db)", + defaultValue = "logs/metrics.db" + ) + private Path databasePath; + + @Option( + names = {"-f", "--format"}, + description = "Output format: table, json, jsonl, csv, tsv, markdown", + defaultValue = "table" + ) + private OutputFormat format = OutputFormat.TABLE; + + @Override + public Integer call() throws Exception { + try (Connection conn = MetricsDatabaseReader.connect(databasePath)) { + SummaryCommand command = new SummaryCommand(); + QueryResult result = command.execute(conn, Map.of()); + + // Use custom markdown formatter for summary command + ResultFormatter formatter; + if (format == OutputFormat.MARKDOWN) { + formatter = new SummaryMarkdownFormatter(); + } else { + formatter = format.createFormatter(); + } + + System.out.println(formatter.format(result)); + + return 0; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + if (System.getenv("DEBUG") != null) { + e.printStackTrace(); + } + return 1; + } + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new SummaryCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/TopKCLI.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/TopKCLI.java new file mode 100644 index 000000000..afb402be1 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/cli/TopKCLI.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.cli; + +import io.nosqlbench.nb.mql.commands.TopKCommand; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.Map; + +/** + * CLI for topk query command. + * Finds top N label sets by metric value. + */ +@Command( + name = "topk", + description = "Find top N label sets by metric value", + mixinStandardHelpOptions = true +) +public class TopKCLI extends AbstractQueryCommandCLI { + + @Option( + names = {"-n", "--top"}, + description = "Number of top results to return", + required = true + ) + private int n; + + @Option( + names = {"-w", "--window"}, + description = "Time window (e.g., 5m, 1h) - optional, defaults to latest snapshot" + ) + private String window; + + @Option( + names = {"--last"}, + description = "Alternative to --window" + ) + private String last; + + @Override + protected MetricsQueryCommand createCommand() { + return new TopKCommand(); + } + + @Override + protected Map buildParams() { + Map params = super.buildParams(); + + params.put("n", n); + + if (window != null) { + params.put("window", window); + } else if (last != null) { + params.put("last", last); + } + + return params; + } + + /** + * Main method for standalone execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new TopKCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/AggregateCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/AggregateCommand.java new file mode 100644 index 000000000..57be0b68c --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/AggregateCommand.java @@ -0,0 +1,318 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.AggregateCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + +/** + * Aggregate command - Perform aggregations (sum, avg, max, min, count) on metrics. + * + * Usage: agg --metric --func [--by ] [--window ] [--labels key=val,...] + * + * Aggregates metric values using SQL aggregate functions, optionally grouping by label dimensions. + * This is the PromQL aggregation operator (sum by, avg by, etc.). + */ +public class AggregateCommand implements MetricsQueryCommand { + + private static final Set VALID_FUNCTIONS = Set.of("sum", "avg", "max", "min", "count"); + + @Override + public String getName() { + return "agg"; + } + + @Override + public String getDescription() { + return "Perform aggregations (sum, avg, max, min, count) on metrics"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + String function = ((String) params.get("function")).toLowerCase(); + @SuppressWarnings("unchecked") + List groupByLabels = (List) params.getOrDefault("by", List.of()); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range (optional - defaults to latest snapshot) + Long startTimestampMs = null; + Long endTimestampMs = null; + boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last"); + + if (params.containsKey("window")) { + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("last")) { + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + // Build the query + String sql = buildQuery(metric, function, groupByLabels, labels, startTimestampMs, endTimestampMs, useLatestOnly); + + // Build columns dynamically based on grouping + List columns = new ArrayList<>(); + columns.addAll(groupByLabels); + columns.add(function); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + + if (!useLatestOnly && startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + + // Add grouped label values + for (String labelName : groupByLabels) { + row.put(labelName, rs.getString(labelName)); + } + + // Add aggregate result + row.put(function, rs.getDouble(function)); + + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + if (!params.containsKey("function")) { + throw new InvalidQueryException("Missing required parameter: function (sum, avg, max, min, count)"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + Object function = params.get("function"); + if (!(function instanceof String)) { + throw new InvalidQueryException("Parameter 'function' must be a string"); + } + + String func = ((String) function).toLowerCase(); + if (!VALID_FUNCTIONS.contains(func)) { + throw new InvalidQueryException( + "Invalid function: " + function + "\n" + + "Valid functions: " + String.join(", ", VALID_FUNCTIONS) + ); + } + + // Validate 'by' parameter if present + if (params.containsKey("by")) { + Object by = params.get("by"); + if (!(by instanceof List)) { + throw new InvalidQueryException("Parameter 'by' must be a List"); + } + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Sum all requests + agg --metric api_requests_total --func sum + + # Average by endpoint + agg --metric api_requests_total --func avg --by endpoint + + # Max latency by operation + agg --metric operation_latency --func max --by operation + + # Count by status code and method + agg --metric api_requests_total --func count --by status,method + + # Sum over last 5 minutes + agg --metric requests_total --func sum --window 5m + """; + } + + private String buildQuery(String metric, String function, List groupByLabels, + Map labelFilters, Long startTs, Long endTs, boolean useLatestOnly) { + StringBuilder sql = new StringBuilder(); + + sql.append("WITH labeled_values AS (\n"); + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE); + + // Add individual label columns for grouping + if (!groupByLabels.isEmpty()) { + sql.append(",\n"); + for (int i = 0; i < groupByLabels.size(); i++) { + sql.append(" lk").append(i).append(".").append(MetricsSchema.COL_LK_NAME).append(" AS label_key_").append(i).append(",\n"); + sql.append(" lv").append(i).append(".").append(MetricsSchema.COL_LV_VALUE).append(" AS label_value_").append(i); + if (i < groupByLabels.size() - 1) { + sql.append(",\n"); + } + } + } + sql.append("\n"); + + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.") + .append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + + if (useLatestOnly) { + sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ") + .append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n"); + } + + // Join with label tables for grouping + for (int i = 0; i < groupByLabels.size(); i++) { + String labelName = groupByLabels.get(i); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm").append(i) + .append(" ON lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_SET_ID) + .append(" = sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk").append(i) + .append(" ON lk").append(i).append(".").append(MetricsSchema.COL_LK_ID) + .append(" = lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_KEY_ID) + .append(" AND lk").append(i).append(".").append(MetricsSchema.COL_LK_NAME).append(" = '").append(labelName).append("'\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv").append(i) + .append(" ON lv").append(i).append(".").append(MetricsSchema.COL_LV_ID) + .append(" = lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + } + + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + + if (useLatestOnly) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n"); + } else if (startTs != null) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + int filterIdx = 0; + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + filterIdx++; + } + + sql.append(")\n"); + + // Aggregation query + sql.append("SELECT\n"); + for (int i = 0; i < groupByLabels.size(); i++) { + sql.append(" label_value_").append(i).append(" AS ").append(groupByLabels.get(i)).append(",\n"); + } + sql.append(" ").append(function.toUpperCase()).append("(value) AS ").append(function).append("\n"); + sql.append("FROM labeled_values\n"); + + if (!groupByLabels.isEmpty()) { + sql.append("GROUP BY "); + for (int i = 0; i < groupByLabels.size(); i++) { + sql.append("label_value_").append(i); + if (i < groupByLabels.size() - 1) { + sql.append(", "); + } + } + sql.append("\n"); + } + + sql.append("ORDER BY ").append(function); + if (function.equals("count") || function.equals("sum")) { + sql.append(" DESC"); // Descending for counts and sums + } + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new AggregateCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/IncreaseCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/IncreaseCommand.java new file mode 100644 index 000000000..21989f9fd --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/IncreaseCommand.java @@ -0,0 +1,249 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.IncreaseCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Increase command - Calculate total increase in counter value over a time window. + * + * Usage: increase --metric --window [--labels key=val,...] + * + * Returns the total increase (max - min) for each label set within the time window. + * This is the standard PromQL increase() function. + * + * Important: Works only with counter metrics (monotonically increasing values). + */ +public class IncreaseCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "increase"; + } + + @Override + public String getDescription() { + return "Calculate total increase in counter over time window"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range + Long startTimestampMs = null; + Long endTimestampMs = null; + + if (params.containsKey("window")) { + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("last")) { + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + // Build the query + String sql = buildQuery(labels, startTimestampMs != null); + + List columns = new ArrayList<>(); + columns.add("increase"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + + if (startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("increase", rs.getDouble("increase")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + // Check that either window OR last is provided + boolean hasWindow = params.containsKey("window"); + boolean hasLast = params.containsKey("last"); + + if (!hasWindow && !hasLast) { + throw new InvalidQueryException( + "Must specify time window using:\n" + + " --window (e.g., 5m, 1h)\n" + + " --last (e.g., 5m, 1h)" + ); + } + + if (hasWindow && hasLast) { + throw new InvalidQueryException( + "Cannot specify both --window and --last. Choose one." + ); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Total requests in last hour + increase --metric api_requests_total --window 1h + + # Total errors in last 5 minutes + increase --metric api_requests_total --window 5m --labels status=500 + + # Operations completed in last day + increase --metric activity_ops_total --window 1d + """; + } + + private String buildQuery(Map labelFilters, boolean hasTimeRange) { + StringBuilder sql = new StringBuilder(); + + sql.append("WITH timeseries AS (\n"); + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME) + .append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.") + .append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n"); + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + + if (hasTimeRange) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n"); + sql.append(")\n"); + + // Calculate increase as MAX - MIN within window for each label set + sql.append("SELECT\n"); + sql.append(" MAX(value) - MIN(value) AS increase,\n"); + sql.append(" labels\n"); + sql.append("FROM timeseries\n"); + sql.append("GROUP BY label_set_id, labels\n"); + sql.append("HAVING MAX(value) > MIN(value)\n"); // Only show increases > 0 + sql.append("ORDER BY labels"); + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution of increase queries. + * Delegates to IncreaseCLI for picocli argument parsing. + * + * Usage: java -cp ... io.nosqlbench.nb.mql.commands.IncreaseCommand --metric ops_total --window 1h + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new IncreaseCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/InstantCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/InstantCommand.java new file mode 100644 index 000000000..ffcb4b47a --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/InstantCommand.java @@ -0,0 +1,192 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.InstantCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Instant query command - retrieves the most recent value for metrics. + * + * Usage: instant --metric [--labels key=val,...] + * + * Returns the latest snapshot value for each unique label set. + */ +public class InstantCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "instant"; + } + + @Override + public String getDescription() { + return "Get the most recent value for a metric"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Build the query + String sql = buildQuery(labels); + + List columns = new ArrayList<>(); + columns.add("timestamp"); + columns.add("value"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("timestamp", rs.getTimestamp("timestamp")); + row.put("value", rs.getDouble("value")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Get latest value for all label sets + instant --metric activity_ops_total + + # Get latest value for specific labels + instant --metric activity_ops_total --labels activity=read,host=server1 + + # Get latest latency + instant --metric operation_latency_ms + """; + } + + private String buildQuery(Map labelFilters) { + StringBuilder sql = new StringBuilder(); + + sql.append("WITH latest_snapshot AS (\n"); + sql.append(" SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append("\n"); + sql.append("),\n"); + + sql.append("labeled_samples AS (\n"); + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME).append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels,\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.").append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n"); + sql.append(" CROSS JOIN latest_snapshot\n"); + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest_snapshot.max_ts\n"); + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.").append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.").append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.").append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n"); + sql.append(")\n"); + + sql.append("SELECT\n"); + sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n"); + sql.append(" value,\n"); + sql.append(" labels\n"); + sql.append("FROM labeled_samples\n"); + sql.append("ORDER BY labels"); + + return sql.toString(); + } + + /** + * Standalone main method for direct execution of instant queries. + * Delegates to InstantCLI for picocli argument parsing. + * + * Usage: java -cp ... io.nosqlbench.nb.mql.commands.InstantCommand --metric ops_total + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new InstantCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/QuantileCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/QuantileCommand.java new file mode 100644 index 000000000..0414274a6 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/QuantileCommand.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.QuantileCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Quantile command - Extract percentile values from timer/summary metrics. + * + * Usage: quantile --metric --p <0.0-1.0> [--window ] [--labels key=val,...] + * + * Retrieves specific quantile values (e.g., p50, p95, p99) from timer or summary metrics. + * Useful for latency analysis and SLA monitoring. + */ +public class QuantileCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "quantile"; + } + + @Override + public String getDescription() { + return "Extract percentile values from timer/summary metrics"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + Double quantile = (Double) params.get("quantile"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range (optional for quantile) + Long startTimestampMs = null; + Long endTimestampMs = null; + + if (params.containsKey("window")) { + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("last")) { + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + // Build the query + String sql = buildQuery(labels, quantile, startTimestampMs != null); + + List columns = new ArrayList<>(); + columns.add("timestamp"); + columns.add("quantile"); + columns.add("quantile_value"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + ps.setDouble(paramIndex++, quantile); + + if (startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("timestamp", rs.getTimestamp("timestamp")); + row.put("quantile", rs.getDouble("quantile")); + row.put("quantile_value", rs.getDouble("quantile_value")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + if (!params.containsKey("quantile")) { + throw new InvalidQueryException("Missing required parameter: quantile (e.g., 0.95 for p95)"); + } + + Object quantile = params.get("quantile"); + if (!(quantile instanceof Double)) { + throw new InvalidQueryException("Parameter 'quantile' must be a number between 0.0 and 1.0"); + } + + double q = (Double) quantile; + if (q < 0.0 || q > 1.0) { + throw new InvalidQueryException("Parameter 'quantile' must be between 0.0 and 1.0, got: " + q); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Get p95 latency for all operations + quantile --metric operation_latency_ms --p 0.95 + + # Get p99 for specific operation + quantile --metric api_latency_ms --p 0.99 --labels operation=select + + # p50 (median) over last 5 minutes + quantile --metric latency_ms --p 0.50 --window 5m + + # Common percentiles + --p 0.50 # p50 (median) + --p 0.95 # p95 + --p 0.99 # p99 + --p 0.999 # p999 + """; + } + + private String buildQuery(Map labelFilters, double quantile, boolean hasTimeRange) { + StringBuilder sql = new StringBuilder(); + + sql.append("SELECT\n"); + sql.append(" datetime(sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" / 1000, 'unixepoch') AS timestamp,\n"); + sql.append(" sq.").append(MetricsSchema.COL_SQ_QUANTILE).append(",\n"); + sql.append(" sq.").append(MetricsSchema.COL_SQ_QUANTILE_VALUE).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME) + .append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n"); + sql.append("FROM ").append(MetricsSchema.TABLE_SAMPLE_QUANTILE).append(" sq\n"); + sql.append("JOIN ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv ON sq.") + .append(MetricsSchema.COL_SQ_SAMPLE_VALUE_ID).append(" = sv.").append(MetricsSchema.COL_SV_ID).append("\n"); + sql.append("JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sv.") + .append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append(" = sn.").append(MetricsSchema.COL_SN_ID).append("\n"); + sql.append("").append(MetricsSchema.joinAllLabels()).append("\n"); + sql.append("WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + sql.append(" AND sq.").append(MetricsSchema.COL_SQ_QUANTILE).append(" = ?\n"); + + if (hasTimeRange) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append("GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append(", sq.") + .append(MetricsSchema.COL_SQ_QUANTILE).append(", sq.").append(MetricsSchema.COL_SQ_QUANTILE_VALUE).append("\n"); + sql.append("ORDER BY sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(", labels"); + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution of quantile queries. + * Delegates to QuantileCLI for picocli argument parsing. + * + * Usage: java -cp ... io.nosqlbench.nb.mql.commands.QuantileCommand --metric latency_ms --p 0.95 + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new QuantileCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RangeCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RangeCommand.java new file mode 100644 index 000000000..9602fa9b9 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RangeCommand.java @@ -0,0 +1,259 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.RangeCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Range query command - retrieves time-series data over a time window. + * + * Usage: range --metric --window [--labels key=val,...] + * range --metric --start --end [--labels key=val,...] + * + * Returns all data points within the specified time range for each label set. + */ +public class RangeCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "range"; + } + + @Override + public String getDescription() { + return "Retrieve time-series data over a time window"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range + Long startTimestampMs = null; + Long endTimestampMs = null; + + if (params.containsKey("window")) { + // Use --window to query last N duration from now + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("start") && params.containsKey("end")) { + // Use explicit start/end timestamps (epoch millis) + startTimestampMs = (Long) params.get("start"); + endTimestampMs = (Long) params.get("end"); + } else if (params.containsKey("last")) { + // Alternative: --last 5m + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + // Build the query + String sql = buildQuery(labels, startTimestampMs != null); + + List columns = new ArrayList<>(); + columns.add("timestamp"); + columns.add("value"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + + if (startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("timestamp", rs.getTimestamp("timestamp")); + row.put("value", rs.getDouble("value")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + // Check that either window OR (start AND end) OR last is provided + boolean hasWindow = params.containsKey("window"); + boolean hasStartEnd = params.containsKey("start") && params.containsKey("end"); + boolean hasLast = params.containsKey("last"); + + if (!hasWindow && !hasStartEnd && !hasLast) { + throw new InvalidQueryException( + "Must specify time range using one of:\n" + + " --window (e.g., 5m, 1h)\n" + + " --last (e.g., 5m, 1h)\n" + + " --start --end (epoch milliseconds)" + ); + } + + if ((hasWindow && hasStartEnd) || (hasWindow && hasLast) || (hasStartEnd && hasLast)) { + throw new InvalidQueryException( + "Cannot specify multiple time range methods. Choose one of: --window, --last, or --start/--end" + ); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Get last 5 minutes of data + range --metric activity_ops_total --window 5m + + # Alternative syntax + range --metric activity_ops_total --last 5m + + # Get data between specific timestamps + range --metric activity_ops_total --start 1729681200000 --end 1729681500000 + + # With label filters + range --metric activity_ops_total --window 10m --labels activity=read + """; + } + + private String buildQuery(Map labelFilters, boolean hasTimeRange) { + StringBuilder sql = new StringBuilder(); + + sql.append("WITH labeled_samples AS (\n"); + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME) + .append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels,\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.") + .append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n"); + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + + if (hasTimeRange) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n"); + sql.append(")\n"); + + sql.append("SELECT\n"); + sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n"); + sql.append(" value,\n"); + sql.append(" labels\n"); + sql.append("FROM labeled_samples\n"); + sql.append("ORDER BY timestamp_ms, labels"); + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution of range queries. + * Delegates to RangeCLI for picocli argument parsing. + * + * Usage: java -cp ... io.nosqlbench.nb.mql.commands.RangeCommand --metric ops_total --window 5m + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new RangeCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RateCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RateCommand.java new file mode 100644 index 000000000..472406ee6 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RateCommand.java @@ -0,0 +1,273 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.RateCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Rate command - Calculate per-second rate of change for counter metrics. + * + * Usage: rate --metric --window [--labels key=val,...] + * + * Calculates the per-second rate by taking the difference between consecutive data points + * and dividing by the time interval between them. This is the standard PromQL rate() function. + * + * Important: Works only with counter metrics (monotonically increasing values). + */ +public class RateCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "rate"; + } + + @Override + public String getDescription() { + return "Calculate per-second rate of change for counters"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range + Long startTimestampMs = null; + Long endTimestampMs = null; + + if (params.containsKey("window")) { + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("last")) { + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + // Build the query + String sql = buildQuery(labels, startTimestampMs != null); + + List columns = new ArrayList<>(); + columns.add("timestamp"); + columns.add("rate_per_sec"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + + if (startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("timestamp", rs.getTimestamp("timestamp")); + row.put("rate_per_sec", rs.getDouble("rate_per_sec")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + // Check that either window OR last is provided (rate needs a time context) + boolean hasWindow = params.containsKey("window"); + boolean hasLast = params.containsKey("last"); + + if (!hasWindow && !hasLast) { + throw new InvalidQueryException( + "Must specify time window using:\n" + + " --window (e.g., 5m, 1h)\n" + + " --last (e.g., 5m, 1h)" + ); + } + + if (hasWindow && hasLast) { + throw new InvalidQueryException( + "Cannot specify both --window and --last. Choose one." + ); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Calculate request rate over last 5 minutes + rate --metric api_requests_total --window 5m + + # Alternative syntax + rate --metric api_requests_total --last 5m + + # With label filters + rate --metric api_requests_total --window 5m --labels method=GET,status=200 + + # Longer window for smoother rate + rate --metric activity_ops_total --window 1h + """; + } + + private String buildQuery(Map labelFilters, boolean hasTimeRange) { + StringBuilder sql = new StringBuilder(); + + // Use window functions to calculate rate between consecutive points + sql.append("WITH timeseries AS (\n"); + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME) + .append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.") + .append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n"); + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + + if (hasTimeRange) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n"); + sql.append(" ORDER BY sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(", sv.") + .append(MetricsSchema.COL_SV_TIMESTAMP_MS).append("\n"); + sql.append("),\n"); + + // Calculate rates using LAG window function + sql.append("rates AS (\n"); + sql.append(" SELECT\n"); + sql.append(" timestamp_ms,\n"); + sql.append(" label_set_id,\n"); + sql.append(" labels,\n"); + sql.append(" value,\n"); + sql.append(" LAG(value) OVER (PARTITION BY label_set_id ORDER BY timestamp_ms) AS prev_value,\n"); + sql.append(" LAG(timestamp_ms) OVER (PARTITION BY label_set_id ORDER BY timestamp_ms) AS prev_timestamp\n"); + sql.append(" FROM timeseries\n"); + sql.append(")\n"); + + sql.append("SELECT\n"); + sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n"); + sql.append(" CASE\n"); + sql.append(" WHEN prev_value IS NULL OR prev_timestamp IS NULL THEN NULL\n"); + sql.append(" WHEN timestamp_ms = prev_timestamp THEN 0.0\n"); + sql.append(" ELSE (value - prev_value) / ((timestamp_ms - prev_timestamp) / 1000.0)\n"); + sql.append(" END AS rate_per_sec,\n"); + sql.append(" labels\n"); + sql.append("FROM rates\n"); + sql.append("WHERE prev_value IS NOT NULL\n"); // Skip first point (no previous value) + sql.append("ORDER BY timestamp_ms, labels"); + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution of rate queries. + * Delegates to RateCLI for picocli argument parsing. + * + * Usage: java -cp ... io.nosqlbench.nb.mql.commands.RateCommand --metric ops_total --window 5m + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new RateCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RatioCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RatioCommand.java new file mode 100644 index 000000000..ab62c8ae3 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/RatioCommand.java @@ -0,0 +1,291 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.RatioCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Ratio command - Calculate ratio between two metrics (e.g., error rate). + * + * Usage: ratio --numerator --denominator [--window ] [--labels key=val,...] + * + * Divides numerator metric by denominator metric for matched label sets. + * Useful for calculating error rates, cache hit ratios, etc. + */ +public class RatioCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "ratio"; + } + + @Override + public String getDescription() { + return "Calculate ratio between two metrics (numerator / denominator)"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String numerator = (String) params.get("numerator"); + String denominator = (String) params.get("denominator"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range (optional - defaults to latest) + Long startTimestampMs = null; + Long endTimestampMs = null; + boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last"); + + if (params.containsKey("window")) { + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("last")) { + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + String sql = buildQuery(numerator, denominator, labels, startTimestampMs, endTimestampMs, useLatestOnly); + + List columns = new ArrayList<>(); + if (!useLatestOnly) { + columns.add("timestamp"); + } + columns.add("ratio"); + columns.add("numerator_value"); + columns.add("denominator_value"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, numerator); + ps.setString(paramIndex++, denominator); + + if (!useLatestOnly && startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters (twice - once for each metric) + for (int i = 0; i < 2; i++) { + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + if (!useLatestOnly) { + row.put("timestamp", rs.getTimestamp("timestamp")); + } + row.put("ratio", rs.getDouble("ratio")); + row.put("numerator_value", rs.getDouble("numerator_value")); + row.put("denominator_value", rs.getDouble("denominator_value")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("numerator")) { + throw new InvalidQueryException("Missing required parameter: numerator"); + } + + if (!params.containsKey("denominator")) { + throw new InvalidQueryException("Missing required parameter: denominator"); + } + + Object numerator = params.get("numerator"); + if (!(numerator instanceof String) || ((String) numerator).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'numerator' must be a non-empty string"); + } + + Object denominator = params.get("denominator"); + if (!(denominator instanceof String) || ((String) denominator).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'denominator' must be a non-empty string"); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Error rate (errors / total requests) + ratio --numerator errors_total --denominator requests_total + + # Cache hit rate + ratio --numerator cache_hits --denominator cache_total + + # Success rate over time + ratio --numerator success_requests --denominator total_requests --window 5m + + # With label filters + ratio --numerator errors --denominator requests --labels endpoint=/api/users + """; + } + + private String buildQuery(String numerator, String denominator, Map labelFilters, + Long startTs, Long endTs, boolean useLatestOnly) { + StringBuilder sql = new StringBuilder(); + + // Build CTEs for numerator and denominator + sql.append("WITH num_data AS (\n"); + sql.append(buildMetricCTE(numerator, "num", labelFilters, startTs, endTs, useLatestOnly)); + sql.append("),\n"); + sql.append("den_data AS (\n"); + sql.append(buildMetricCTE(denominator, "den", labelFilters, startTs, endTs, useLatestOnly)); + sql.append(")\n"); + + // Join and calculate ratio + sql.append("SELECT\n"); + if (!useLatestOnly) { + sql.append(" num_data.timestamp,\n"); + } + sql.append(" CASE\n"); + sql.append(" WHEN den_data.value = 0 THEN NULL\n"); + sql.append(" ELSE num_data.value / den_data.value\n"); + sql.append(" END AS ratio,\n"); + sql.append(" num_data.value AS numerator_value,\n"); + sql.append(" den_data.value AS denominator_value,\n"); + sql.append(" num_data.labels\n"); + sql.append("FROM num_data\n"); + sql.append("JOIN den_data ON num_data.label_set_id = den_data.label_set_id"); + + if (!useLatestOnly) { + sql.append(" AND num_data.timestamp_ms = den_data.timestamp_ms"); + } + + sql.append("\nORDER BY "); + if (!useLatestOnly) { + sql.append("num_data.timestamp_ms, "); + } + sql.append("num_data.labels"); + + return sql.toString(); + } + + private String buildMetricCTE(String metric, String alias, Map labelFilters, + Long startTs, Long endTs, boolean useLatestOnly) { + StringBuilder sql = new StringBuilder(); + + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME) + .append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.") + .append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n"); + + if (useLatestOnly) { + sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ") + .append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n"); + } + + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + + if (useLatestOnly) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n"); + } else if (startTs != null) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID); + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new RatioCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SessionCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SessionCommand.java new file mode 100644 index 000000000..57936c1d2 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SessionCommand.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.SessionCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Session command - Display session time information. + * + * Usage: session + * + * Shows timing information about the metrics session including first/last snapshots, + * duration, and snapshot cadence. Useful for understanding the data timeline. + */ +public class SessionCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "session"; + } + + @Override + public String getDescription() { + return "Display session time information and snapshot cadence"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + long startTime = System.currentTimeMillis(); + + String sql = """ + SELECT + MIN(timestamp_ms) as first_snapshot_ms, + MAX(timestamp_ms) as last_snapshot_ms, + COUNT(DISTINCT timestamp_ms) as total_snapshots, + COUNT(*) as total_samples + FROM sample_value + """; + + List columns = new ArrayList<>(); + columns.add("first_snapshot"); + columns.add("last_snapshot"); + columns.add("duration"); + columns.add("total_snapshots"); + columns.add("total_samples"); + columns.add("avg_interval"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + if (rs.next()) { + long firstMs = rs.getLong("first_snapshot_ms"); + long lastMs = rs.getLong("last_snapshot_ms"); + int totalSnapshots = rs.getInt("total_snapshots"); + int totalSamples = rs.getInt("total_samples"); + + long durationMs = lastMs - firstMs; + long avgIntervalMs = totalSnapshots > 1 ? durationMs / (totalSnapshots - 1) : 0; + + Map row = new LinkedHashMap<>(); + row.put("first_snapshot", new Timestamp(firstMs).toString()); + row.put("last_snapshot", new Timestamp(lastMs).toString()); + row.put("duration", formatDuration(durationMs)); + row.put("total_snapshots", totalSnapshots); + row.put("total_samples", totalSamples); + row.put("avg_interval", formatDuration(avgIntervalMs)); + + rows.add(row); + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + // No parameters required for session command + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Show session information + session + + # JSON format for scripting + session --format json + + # Markdown for documentation + session --format markdown + """; + } + + private String formatDuration(long millis) { + if (millis == 0) { + return "0s"; + } + + long seconds = millis / 1000; + long minutes = seconds / 60; + long hours = minutes / 60; + long days = hours / 24; + + if (days > 0) { + return String.format("%dd %dh %dm", days, hours % 24, minutes % 60); + } else if (hours > 0) { + return String.format("%dh %dm %ds", hours, minutes % 60, seconds % 60); + } else if (minutes > 0) { + return String.format("%dm %ds", minutes, seconds % 60); + } else { + return String.format("%ds", seconds); + } + } + + /** + * Standalone main method for direct execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new SessionCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SqlCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SqlCommand.java new file mode 100644 index 000000000..ee59143bf --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SqlCommand.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.SqlCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * SQL command - Execute raw SQLite queries against the metrics database. + * + * Usage: sql --query "SELECT * FROM metric_family" + * + * Allows advanced users to write custom SQL queries for exploration, + * debugging, and analytics not covered by high-level commands. + * + * Safety: + * - Connection is read-only (no modifications possible) + * - Query timeout prevents runaway queries + * - Full SQLite dialect and functions available + */ +public class SqlCommand implements MetricsQueryCommand { + + private static final int DEFAULT_QUERY_TIMEOUT_SECONDS = 30; + + @Override + public String getName() { + return "sql"; + } + + @Override + public String getDescription() { + return "Execute raw SQLite queries (read-only)"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String sql = (String) params.get("query"); + Integer timeoutSeconds = (Integer) params.getOrDefault("timeout", DEFAULT_QUERY_TIMEOUT_SECONDS); + + long startTime = System.currentTimeMillis(); + + List columns = new ArrayList<>(); + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setQueryTimeout(timeoutSeconds); + + try (ResultSet rs = ps.executeQuery()) { + // Extract column names from result set metadata + ResultSetMetaData metadata = rs.getMetaData(); + int columnCount = metadata.getColumnCount(); + + for (int i = 1; i <= columnCount; i++) { + columns.add(metadata.getColumnName(i)); + } + + // Extract rows + while (rs.next()) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= columnCount; i++) { + String columnName = metadata.getColumnName(i); + Object value = rs.getObject(i); + row.put(columnName, value); + } + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("query")) { + throw new InvalidQueryException("Missing required parameter: query"); + } + + Object query = params.get("query"); + if (!(query instanceof String) || ((String) query).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'query' must be a non-empty SQL string"); + } + + String sql = ((String) query).trim().toUpperCase(); + + // Basic validation: ensure it's a SELECT query (read-only) + if (!sql.startsWith("SELECT") && !sql.startsWith("WITH") && !sql.startsWith("PRAGMA")) { + throw new InvalidQueryException( + "Only SELECT, WITH (CTE), and PRAGMA queries are allowed.\n" + + "Database is read-only to prevent modifications." + ); + } + + // Block dangerous pragmas + if (sql.contains("PRAGMA WRITABLE_SCHEMA") || sql.contains("PRAGMA CASE_SENSITIVE_LIKE")) { + throw new InvalidQueryException("This PRAGMA is not allowed"); + } + + // Validate timeout if present + if (params.containsKey("timeout")) { + Object timeout = params.get("timeout"); + if (!(timeout instanceof Integer) || (Integer) timeout <= 0) { + throw new InvalidQueryException("Parameter 'timeout' must be a positive integer (seconds)"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # List all metric families + sql --query "SELECT name, type, unit FROM metric_family ORDER BY name" + + # Get sample counts by metric + sql --query "SELECT sn.sample, COUNT(*) as count FROM sample_value sv JOIN sample_name sn ON sv.sample_name_id = sn.id GROUP BY sn.sample" + + # Custom time-series query + sql --query "SELECT datetime(timestamp_ms/1000, 'unixepoch') as time, value FROM sample_value WHERE sample_name_id = 1 ORDER BY timestamp_ms" + + # Schema inspection + sql --query "PRAGMA table_info(sample_value)" + + # With timeout + sql --query "SELECT * FROM sample_value" --timeout 60 + """; + } + + /** + * Standalone main method for direct execution of SQL queries. + * Delegates to SqlCLI for picocli argument parsing. + * + * Usage: java -cp ... io.nosqlbench.nb.mql.commands.SqlCommand --query "SELECT ..." + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new SqlCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SummaryCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SummaryCommand.java new file mode 100644 index 000000000..6c9529621 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/SummaryCommand.java @@ -0,0 +1,353 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.SummaryCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.*; + +/** + * Summary command - Display comprehensive end-of-session metrics report. + * + * Usage: summary + * + * Provides an elaborate session summary focusing on NoSQLBench core metrics: + * - Session timeline and duration + * - Core operation metrics (cycles, result, tries, errors) + * - Activity breakdowns + * - Error analysis + * - Performance indicators + * + * Designed to give a complete picture of the session at a glance. + */ +public class SummaryCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "summary"; + } + + @Override + public String getDescription() { + return "Display comprehensive end-of-session metrics report"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + long startTime = System.currentTimeMillis(); + + // Build comprehensive summary with multiple sections + List columns = new ArrayList<>(); + columns.add("section"); + columns.add("metric"); + columns.add("value"); + columns.add("details"); + + List> rows = new ArrayList<>(); + + // Section 1: Session Overview + addSessionOverview(conn, rows); + + // Section 2: Core Metrics Summary (cycles, result, tries, errors) + addCoreMetricsSummary(conn, rows); + + // Section 3: Activity Breakdown + addActivityBreakdown(conn, rows); + + // Section 4: Error Analysis + addErrorAnalysis(conn, rows); + + // Section 5: All Available Metrics + addAllMetrics(conn, rows); + + long executionTime = System.currentTimeMillis() - startTime; + + // Build a single SQL representing all queries (for reference) + String combinedSql = "-- Multiple queries executed for comprehensive summary\n" + + "-- See SummaryCommand.java for details"; + + return new QueryResult(columns, rows, combinedSql, executionTime); + } + + private void addSessionOverview(Connection conn, List> rows) throws SQLException { + String sql = """ + SELECT + MIN(timestamp_ms) as first_ts, + MAX(timestamp_ms) as last_ts, + COUNT(DISTINCT timestamp_ms) as snapshots, + COUNT(*) as total_samples + FROM sample_value + """; + + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + if (rs.next()) { + long firstMs = rs.getLong("first_ts"); + long lastMs = rs.getLong("last_ts"); + int snapshots = rs.getInt("snapshots"); + int totalSamples = rs.getInt("total_samples"); + long durationMs = lastMs - firstMs; + + rows.add(createRow("SESSION", "Time Range", + new Timestamp(firstMs) + " to " + new Timestamp(lastMs), + formatDuration(durationMs))); + + rows.add(createRow("SESSION", "Snapshots", String.valueOf(snapshots), + "Total samples: " + totalSamples)); + + if (snapshots > 1) { + long avgInterval = durationMs / (snapshots - 1); + rows.add(createRow("SESSION", "Avg Interval", formatDuration(avgInterval), + "~" + (avgInterval / 1000) + "s cadence")); + } + } + } + } + + private void addCoreMetricsSummary(Connection conn, List> rows) throws SQLException { + // Query for core NoSQLBench metrics: cycles, result, tries, errors + String[] coreMetrics = {"cycles", "result", "tries", "errors"}; + + for (String metricBase : coreMetrics) { + String sql = """ + SELECT + SUM(sv.value) as total_value, + COUNT(DISTINCT sv.label_set_id) as label_sets + FROM sample_value sv + JOIN sample_name sn ON sv.sample_name_id = sn.id + WHERE sn.sample LIKE ? + AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) + """; + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, "%" + metricBase + "%"); + + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + double total = rs.getDouble("total_value"); + int labelSets = rs.getInt("label_sets"); + + if (labelSets > 0) { + rows.add(createRow("CORE METRICS", metricBase.toUpperCase(), + String.format("%.0f", total), + labelSets + " label set(s)")); + } + } + } + } + } + + // Calculate error rate if we have both result and errors + calculateErrorRate(conn, rows); + } + + private void calculateErrorRate(Connection conn, List> rows) throws SQLException { + String sql = """ + SELECT + SUM(CASE WHEN sn.sample LIKE '%result%' THEN sv.value ELSE 0 END) as results, + SUM(CASE WHEN sn.sample LIKE '%error%' THEN sv.value ELSE 0 END) as errors + FROM sample_value sv + JOIN sample_name sn ON sv.sample_name_id = sn.id + WHERE sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) + """; + + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + if (rs.next()) { + double results = rs.getDouble("results"); + double errors = rs.getDouble("errors"); + + if (results > 0) { + double errorRate = (errors / results) * 100.0; + rows.add(createRow("CORE METRICS", "Error Rate", + String.format("%.2f%%", errorRate), + String.format("%.0f errors / %.0f results", errors, results))); + } + } + } + } + + private void addActivityBreakdown(Connection conn, List> rows) throws SQLException { + // Group by activity label if it exists + String sql = """ + SELECT + lv.value as activity, + SUM(sv.value) as total + FROM sample_value sv + JOIN sample_name sn ON sv.sample_name_id = sn.id + JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id + JOIN label_key lk ON lsm.label_key_id = lk.id + JOIN label_value lv ON lsm.label_value_id = lv.id + WHERE lk.name = 'activity' + AND sn.sample LIKE '%ops%' + AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) + GROUP BY lv.value + ORDER BY total DESC + """; + + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + while (rs.next()) { + String activity = rs.getString("activity"); + double total = rs.getDouble("total"); + + rows.add(createRow("ACTIVITY", activity, + String.format("%.0f", total), "operations")); + } + } + } + + private void addErrorAnalysis(Connection conn, List> rows) throws SQLException { + String sql = """ + SELECT + sn.sample, + SUM(sv.value) as total_errors + FROM sample_value sv + JOIN sample_name sn ON sv.sample_name_id = sn.id + WHERE sn.sample LIKE '%error%' + AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) + GROUP BY sn.sample + HAVING total_errors > 0 + ORDER BY total_errors DESC + """; + + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + while (rs.next()) { + String sample = rs.getString("sample"); + double errors = rs.getDouble("total_errors"); + + rows.add(createRow("ERRORS", sample, + String.format("%.0f", errors), "total errors")); + } + } + } + + private void addAllMetrics(Connection conn, List> rows) throws SQLException { + String sql = """ + SELECT + mf.name AS metric_name, + mf.type AS metric_type, + COUNT(DISTINCT sv.label_set_id) AS unique_label_sets, + COUNT(*) AS total_samples, + MIN(sv.value) AS min_value, + MAX(sv.value) AS max_value + FROM metric_family mf + JOIN sample_name sn ON sn.metric_family_id = mf.id + JOIN sample_value sv ON sv.sample_name_id = sn.id + GROUP BY mf.id, mf.name, mf.type + ORDER BY mf.name + """; + + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + + while (rs.next()) { + String name = rs.getString("metric_name"); + String type = rs.getString("metric_type"); + int labelSets = rs.getInt("unique_label_sets"); + int samples = rs.getInt("total_samples"); + double min = rs.getDouble("min_value"); + double max = rs.getDouble("max_value"); + + String details = String.format("type=%s, samples=%d, label_sets=%d, range=[%.1f, %.1f]", + type, samples, labelSets, min, max); + + rows.add(createRow("ALL METRICS", name, "", details)); + } + } + } + + private Map createRow(String section, String metric, String value, String details) { + Map row = new LinkedHashMap<>(); + row.put("section", section); + row.put("metric", metric); + row.put("value", value); + row.put("details", details); + return row; + } + + private String formatDuration(long millis) { + if (millis == 0) { + return "0s"; + } + + long seconds = millis / 1000; + long minutes = seconds / 60; + long hours = minutes / 60; + long days = hours / 24; + + if (days > 0) { + return String.format("%dd %dh %dm", days, hours % 24, minutes % 60); + } else if (hours > 0) { + return String.format("%dh %dm %ds", hours, minutes % 60, seconds % 60); + } else if (minutes > 0) { + return String.format("%dm %ds", minutes, seconds % 60); + } else { + return String.format("%ds", seconds); + } + } + + @Override + public void validate(Map params) throws InvalidQueryException { + // No parameters required for summary command + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Show comprehensive session summary + summary + + # JSON format for scripting + summary --format json + + # Markdown for reports + summary --format markdown + + # Use at end of session + nb5 run.yaml && nb5 mql summary + """; + } + + /** + * Standalone main method for direct execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new SummaryCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/TopKCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/TopKCommand.java new file mode 100644 index 000000000..403b92234 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/commands/TopKCommand.java @@ -0,0 +1,243 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.cli.TopKCLI; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.MetricsQueryCommand; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.query.TimeWindowParser; +import io.nosqlbench.nb.mql.schema.MetricsSchema; +import picocli.CommandLine; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * TopK command - Find top N label sets by metric value. + * + * Usage: topk --metric --n [--window ] [--labels key=val,...] + * + * Returns the N label sets with highest metric values. + * This is the PromQL topk() function. + */ +public class TopKCommand implements MetricsQueryCommand { + + @Override + public String getName() { + return "topk"; + } + + @Override + public String getDescription() { + return "Find top N label sets by metric value"; + } + + @Override + public QueryResult execute(Connection conn, Map params) + throws SQLException, InvalidQueryException { + + validate(params); + + String metric = (String) params.get("metric"); + Integer n = (Integer) params.get("n"); + @SuppressWarnings("unchecked") + Map labels = (Map) params.getOrDefault("labels", Map.of()); + + long startTime = System.currentTimeMillis(); + + // Determine time range (optional - defaults to latest) + Long startTimestampMs = null; + Long endTimestampMs = null; + boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last"); + + if (params.containsKey("window")) { + String window = (String) params.get("window"); + long windowMs = TimeWindowParser.parseToMillis(window); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } else if (params.containsKey("last")) { + String last = (String) params.get("last"); + long windowMs = TimeWindowParser.parseToMillis(last); + endTimestampMs = getCurrentMaxTimestamp(conn); + startTimestampMs = endTimestampMs - windowMs; + } + + String sql = buildQuery(labels, startTimestampMs, endTimestampMs, useLatestOnly, n); + + List columns = new ArrayList<>(); + columns.add("value"); + columns.add("labels"); + + List> rows = new ArrayList<>(); + + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int paramIndex = 1; + ps.setString(paramIndex++, metric); + + if (!useLatestOnly && startTimestampMs != null) { + ps.setLong(paramIndex++, startTimestampMs); + ps.setLong(paramIndex++, endTimestampMs); + } + + // Add label filter parameters + for (Map.Entry label : labels.entrySet()) { + ps.setString(paramIndex++, label.getKey()); + ps.setString(paramIndex++, label.getValue()); + } + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map row = new LinkedHashMap<>(); + row.put("value", rs.getDouble("value")); + row.put("labels", rs.getString("labels")); + rows.add(row); + } + } + } + + long executionTime = System.currentTimeMillis() - startTime; + + return new QueryResult(columns, rows, sql, executionTime); + } + + @Override + public void validate(Map params) throws InvalidQueryException { + if (!params.containsKey("metric")) { + throw new InvalidQueryException("Missing required parameter: metric"); + } + + if (!params.containsKey("n")) { + throw new InvalidQueryException("Missing required parameter: n (number of top results)"); + } + + Object metric = params.get("metric"); + if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) { + throw new InvalidQueryException("Parameter 'metric' must be a non-empty string"); + } + + Object n = params.get("n"); + if (!(n instanceof Integer) || (Integer) n <= 0) { + throw new InvalidQueryException("Parameter 'n' must be a positive integer"); + } + + // Validate labels if present + if (params.containsKey("labels")) { + Object labels = params.get("labels"); + if (!(labels instanceof Map)) { + throw new InvalidQueryException("Parameter 'labels' must be a Map"); + } + } + } + + @Override + public String getUsageExamples() { + return """ + Examples: + # Top 5 endpoints by request count + topk --metric api_requests_total --n 5 + + # Top 3 slowest operations + topk --metric operation_latency --n 3 + + # Top 10 in last hour + topk --metric requests_total --n 10 --window 1h + + # Top 5 errors + topk --metric errors_total --n 5 --labels status=500 + """; + } + + private String buildQuery(Map labelFilters, Long startTs, Long endTs, + boolean useLatestOnly, int limit) { + StringBuilder sql = new StringBuilder(); + + sql.append("WITH latest_values AS (\n"); + sql.append(" SELECT\n"); + sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n"); + sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME) + .append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.") + .append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n"); + sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n"); + + if (useLatestOnly) { + sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ") + .append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n"); + } + + sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n"); + + if (useLatestOnly) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n"); + } else if (startTs != null) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n"); + sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n"); + } + + // Add label filters + for (String labelKey : labelFilters.keySet()) { + sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n"); + sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n"); + sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.") + .append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n"); + sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.") + .append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n"); + sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.") + .append(MetricsSchema.COL_LV_VALUE).append(" = ?\n"); + sql.append(" )\n"); + } + + sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n"); + sql.append(")\n"); + + sql.append("SELECT value, labels\n"); + sql.append("FROM latest_values\n"); + sql.append("ORDER BY value DESC\n"); + sql.append("LIMIT ").append(limit); + + return sql.toString(); + } + + private long getCurrentMaxTimestamp(Connection conn) throws SQLException { + String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " + + MetricsSchema.TABLE_SAMPLE_VALUE; + try (PreparedStatement ps = conn.prepareStatement(sql); + ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return rs.getLong(1); + } + throw new SQLException("No data in database"); + } + } + + /** + * Standalone main method for direct execution. + */ + public static void main(String[] args) { + int exitCode = new CommandLine(new TopKCLI()).execute(args); + System.exit(exitCode); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/CsvFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/CsvFormatter.java new file mode 100644 index 000000000..0af246740 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/CsvFormatter.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import io.nosqlbench.nb.mql.query.QueryResult; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Formats query results as CSV for spreadsheet import. + */ +public class CsvFormatter implements ResultFormatter { + + private final boolean includeHeaders; + + public CsvFormatter() { + this(true); + } + + public CsvFormatter(boolean includeHeaders) { + this.includeHeaders = includeHeaders; + } + + @Override + public String format(QueryResult result) { + StringWriter writer = new StringWriter(); + + try { + CSVFormat format = includeHeaders + ? CSVFormat.DEFAULT.builder().setHeader(result.columns().toArray(new String[0])).build() + : CSVFormat.DEFAULT; + + try (CSVPrinter printer = new CSVPrinter(writer, format)) { + for (Map row : result.rows()) { + List values = new ArrayList<>(); + for (String column : result.columns()) { + values.add(row.get(column)); + } + printer.printRecord(values); + } + } + } catch (IOException e) { + throw new RuntimeException("Failed to format CSV", e); + } + + return writer.toString(); + } + + @Override + public String getName() { + return "csv"; + } + + @Override + public String getFileExtension() { + return "csv"; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/JsonFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/JsonFormatter.java new file mode 100644 index 000000000..ce58a3992 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/JsonFormatter.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.nosqlbench.nb.mql.query.QueryResult; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Formats query results as JSON for programmatic consumption. + */ +public class JsonFormatter implements ResultFormatter { + + private final Gson gson; + private final boolean pretty; + + public JsonFormatter() { + this(false); + } + + public JsonFormatter(boolean pretty) { + this.pretty = pretty; + GsonBuilder builder = new GsonBuilder(); + if (pretty) { + builder.setPrettyPrinting(); + } + this.gson = builder.create(); + } + + @Override + public String format(QueryResult result) { + Map output = new LinkedHashMap<>(); + output.put("columns", result.columns()); + output.put("rows", result.rows()); + output.put("rowCount", result.rowCount()); + output.put("executionTimeMs", result.executionTimeMs()); + + return gson.toJson(output); + } + + @Override + public String getName() { + return "json"; + } + + @Override + public String getFileExtension() { + return "json"; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/JsonLinesFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/JsonLinesFormatter.java new file mode 100644 index 000000000..aed7cf7f4 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/JsonLinesFormatter.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import com.google.gson.Gson; +import io.nosqlbench.nb.mql.query.QueryResult; + +import java.util.Map; + +/** + * Formats query results as JSONL (JSON Lines) for streaming data processing. + * Each row is output as a single JSON object on its own line, with no array wrapper. + * This format is efficient for large result sets and compatible with streaming processors. + */ +public class JsonLinesFormatter implements ResultFormatter { + + private final Gson gson; + + public JsonLinesFormatter() { + // Compact JSON (no pretty printing for JSONL) + this.gson = new Gson(); + } + + @Override + public String format(QueryResult result) { + StringBuilder sb = new StringBuilder(); + + // Each row is a separate JSON object on its own line + for (Map row : result.rows()) { + sb.append(gson.toJson(row)); + sb.append("\n"); + } + + return sb.toString(); + } + + @Override + public String getName() { + return "jsonl"; + } + + @Override + public String getFileExtension() { + return "jsonl"; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/MarkdownFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/MarkdownFormatter.java new file mode 100644 index 000000000..c044dd8ec --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/MarkdownFormatter.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import io.nosqlbench.nb.mql.query.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Formats query results as GitHub-flavored Markdown tables. + * Output can be directly copy-pasted into markdown documentation. + */ +public class MarkdownFormatter implements ResultFormatter { + + private static final int DEFAULT_MAX_COLUMN_WIDTH = 50; + + @Override + public String format(QueryResult result) { + if (result.isEmpty()) { + return "*No results found.*\n"; + } + + List columns = result.columns(); + List> rows = result.rows(); + + // Calculate column widths + Map columnWidths = calculateColumnWidths(columns, rows); + + StringBuilder sb = new StringBuilder(); + + // Header + sb.append("| "); + for (int i = 0; i < columns.size(); i++) { + String column = columns.get(i); + sb.append(padRight(column, columnWidths.get(column))); + if (i < columns.size() - 1) { + sb.append(" | "); + } + } + sb.append(" |\n"); + + // Separator row + sb.append("|"); + for (int i = 0; i < columns.size(); i++) { + String column = columns.get(i); + sb.append("-".repeat(columnWidths.get(column) + 2)); // +2 for padding spaces + if (i < columns.size() - 1) { + sb.append("|"); + } + } + sb.append("|\n"); + + // Data rows + for (Map row : rows) { + sb.append("| "); + for (int i = 0; i < columns.size(); i++) { + String column = columns.get(i); + Object value = row.get(column); + String strValue = value != null ? escapeMarkdown(value.toString()) : ""; + + // Truncate if too long + if (strValue.length() > DEFAULT_MAX_COLUMN_WIDTH) { + strValue = strValue.substring(0, DEFAULT_MAX_COLUMN_WIDTH - 3) + "..."; + } + + sb.append(padRight(strValue, columnWidths.get(column))); + if (i < columns.size() - 1) { + sb.append(" | "); + } + } + sb.append(" |\n"); + } + + // Footer with metadata + sb.append("\n"); + sb.append("*").append(result.rowCount()).append(" row"); + if (result.rowCount() != 1) { + sb.append("s"); + } + sb.append(" (").append(result.executionTimeMs()).append("ms)*\n"); + + return sb.toString(); + } + + @Override + public String getName() { + return "markdown"; + } + + @Override + public String getFileExtension() { + return "md"; + } + + private Map calculateColumnWidths(List columns, List> rows) { + Map widths = new java.util.LinkedHashMap<>(); + + // Initialize with header widths + for (String column : columns) { + widths.put(column, column.length()); + } + + // Check data widths + for (Map row : rows) { + for (String column : columns) { + Object value = row.get(column); + String strValue = value != null ? value.toString() : ""; + int currentWidth = widths.get(column); + int valueWidth = Math.min(strValue.length(), DEFAULT_MAX_COLUMN_WIDTH); + widths.put(column, Math.max(currentWidth, valueWidth)); + } + } + + return widths; + } + + private String padRight(String str, int width) { + if (str.length() >= width) { + return str; + } + return str + " ".repeat(width - str.length()); + } + + private String escapeMarkdown(String str) { + // Escape pipe characters which would break the table + return str.replace("|", "\\|"); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/OutputFormat.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/OutputFormat.java new file mode 100644 index 000000000..9238592ba --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/OutputFormat.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +/** + * Enum representing all supported output formats for query results. + * Provides type safety and validation for format selection in CLI. + */ +public enum OutputFormat { + /** + * ASCII tables for console output (default). + */ + TABLE("table", "txt", "ASCII tables for console output") { + @Override + public ResultFormatter createFormatter() { + return new TableFormatter(); + } + }, + + /** + * Comma-separated values for spreadsheet import. + */ + CSV("csv", "csv", "Comma-separated values for Excel/Sheets") { + @Override + public ResultFormatter createFormatter() { + return new CsvFormatter(); + } + }, + + /** + * Tab-separated values for Unix text processing. + */ + TSV("tsv", "tsv", "Tab-separated values for awk/cut/paste") { + @Override + public ResultFormatter createFormatter() { + return new TsvFormatter(); + } + }, + + /** + * Pretty-printed JSON for programmatic use. + */ + JSON("json", "json", "Pretty-printed JSON with metadata") { + @Override + public ResultFormatter createFormatter() { + return new JsonFormatter(true); + } + }, + + /** + * JSON Lines (one JSON object per line) for streaming. + */ + JSONL("jsonl", "jsonl", "JSON Lines for streaming processors") { + @Override + public ResultFormatter createFormatter() { + return new JsonLinesFormatter(); + } + }, + + /** + * GitHub-flavored Markdown tables for documentation. + */ + MARKDOWN("markdown", "md", "Markdown tables for documentation") { + @Override + public ResultFormatter createFormatter() { + return new MarkdownFormatter(); + } + }; + + private final String name; + private final String extension; + private final String description; + + OutputFormat(String name, String extension, String description) { + this.name = name; + this.extension = extension; + this.description = description; + } + + /** + * Create a formatter instance for this format. + * Each enum value knows how to create its own formatter. + * + * @return ResultFormatter instance + */ + public abstract ResultFormatter createFormatter(); + + /** + * Get the format name as it appears in CLI arguments. + */ + public String getName() { + return name; + } + + /** + * Get the file extension for this format. + */ + public String getExtension() { + return extension; + } + + /** + * Get a human-readable description. + */ + public String getDescription() { + return description; + } + + /** + * Parse a format name string to an enum value. + * + * @param name Format name (case-insensitive) + * @return The OutputFormat enum, or null if not found + */ + public static OutputFormat fromString(String name) { + if (name == null || name.trim().isEmpty()) { + return TABLE; // Default + } + + String normalized = name.toLowerCase().trim(); + for (OutputFormat format : values()) { + if (format.name.equals(normalized)) { + return format; + } + } + return null; // Not found + } + + /** + * Get a comma-separated list of all format names. + * Useful for CLI help text. + */ + public static String getAllFormatNames() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < values().length; i++) { + sb.append(values()[i].name); + if (i < values().length - 1) { + sb.append(", "); + } + } + return sb.toString(); + } + + @Override + public String toString() { + return name; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/ResultFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/ResultFormatter.java new file mode 100644 index 000000000..5eb7210b4 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/ResultFormatter.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import io.nosqlbench.nb.mql.query.QueryResult; + +/** + * Interface for formatting query results in different output formats. + */ +public interface ResultFormatter { + + /** + * Format a query result as a string. + * + * @param result The query result to format + * @return Formatted string representation + */ + String format(QueryResult result); + + /** + * Get the name of this formatter (e.g., "table", "json", "csv"). + * + * @return Formatter name + */ + String getName(); + + /** + * Get the file extension for this format (e.g., "txt", "json", "csv"). + * + * @return File extension without the dot + */ + String getFileExtension(); +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/SummaryMarkdownFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/SummaryMarkdownFormatter.java new file mode 100644 index 000000000..48730a205 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/SummaryMarkdownFormatter.java @@ -0,0 +1,221 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import io.nosqlbench.nb.mql.query.QueryResult; + +import java.util.*; + +/** + * Custom markdown formatter for summary command output. + * Creates a nicely structured report with tree views and sections. + */ +public class SummaryMarkdownFormatter implements ResultFormatter { + + @Override + public String format(QueryResult result) { + if (result.isEmpty()) { + return "*No data available.*\n"; + } + + StringBuilder md = new StringBuilder(); + + // Title + md.append("# NoSQLBench Session Summary\n\n"); + + // Group rows by section + Map>> sections = new LinkedHashMap<>(); + for (Map row : result.rows()) { + String section = (String) row.get("section"); + sections.computeIfAbsent(section, k -> new ArrayList<>()).add(row); + } + + // Render each section + for (Map.Entry>> entry : sections.entrySet()) { + String sectionName = entry.getKey(); + List> sectionRows = entry.getValue(); + + if ("SESSION".equals(sectionName)) { + renderSessionSection(md, sectionRows); + } else if ("CORE METRICS".equals(sectionName)) { + renderCoreMetricsSection(md, sectionRows); + } else if ("ACTIVITY".equals(sectionName)) { + renderActivitySection(md, sectionRows); + } else if ("ERRORS".equals(sectionName)) { + renderErrorsSection(md, sectionRows); + } else if ("ALL METRICS".equals(sectionName)) { + renderAllMetricsTree(md, sectionRows); + } else { + renderGenericSection(md, sectionName, sectionRows); + } + + md.append("\n"); + } + + // Footer + md.append("---\n\n"); + md.append(String.format("*Report generated in %dms*\n", result.executionTimeMs())); + + return md.toString(); + } + + private void renderSessionSection(StringBuilder md, List> rows) { + md.append("## 📊 Session Overview\n\n"); + + for (Map row : rows) { + String metric = (String) row.get("metric"); + String value = (String) row.get("value"); + String details = (String) row.get("details"); + + md.append("**").append(metric).append(":** \n"); + md.append("`").append(value).append("`"); + if (details != null && !details.isEmpty()) { + md.append(" — *").append(details).append("*"); + } + md.append("\n\n"); + } + } + + private void renderCoreMetricsSection(StringBuilder md, List> rows) { + md.append("## 🎯 Core Metrics\n\n"); + + md.append("| Metric | Value | Details |\n"); + md.append("|--------|-------|----------|\n"); + + for (Map row : rows) { + String metric = (String) row.get("metric"); + String value = (String) row.get("value"); + String details = (String) row.get("details"); + + md.append("| **").append(metric).append("** | "); + md.append("`").append(value).append("` | "); + md.append(details != null ? details : ""); + md.append(" |\n"); + } + + md.append("\n"); + } + + private void renderActivitySection(StringBuilder md, List> rows) { + md.append("## 📈 Activity Breakdown\n\n"); + + md.append("```\n"); + long total = rows.stream() + .mapToLong(row -> { + String value = (String) row.get("value"); + return (long) Double.parseDouble(value); + }) + .sum(); + + for (Map row : rows) { + String activity = (String) row.get("metric"); + String value = (String) row.get("value"); + long count = (long) Double.parseDouble(value); + double percentage = (count * 100.0) / total; + + md.append(String.format("%-15s %,12d ops (%5.1f%%)\n", + activity + ":", count, percentage)); + } + + md.append(String.format("%-15s %,12d ops\n", "TOTAL:", total)); + md.append("```\n\n"); + } + + private void renderErrorsSection(StringBuilder md, List> rows) { + md.append("## ⚠️ Errors\n\n"); + + if (rows.isEmpty()) { + md.append("*No errors recorded.*\n\n"); + return; + } + + md.append("| Error Type | Count |\n"); + md.append("|------------|-------|\n"); + + for (Map row : rows) { + String metric = (String) row.get("metric"); + String value = (String) row.get("value"); + + md.append("| `").append(metric).append("` | "); + md.append("**").append(value).append("** |\n"); + } + + md.append("\n"); + } + + private void renderAllMetricsTree(StringBuilder md, List> rows) { + md.append("## 📋 Metrics Inventory\n\n"); + + md.append("```\n"); + md.append("Metrics Database\n"); + + for (int i = 0; i < rows.size(); i++) { + Map row = rows.get(i); + String name = (String) row.get("metric"); + String details = (String) row.get("details"); + + boolean isLast = (i == rows.size() - 1); + String prefix = isLast ? "└── " : "├── "; + + md.append(prefix).append(name).append("\n"); + + if (details != null && !details.isEmpty()) { + String detailPrefix = isLast ? " " : "│ "; + // Parse details and format as tree branches + // Handle range=[min, max] specially to keep it on one line + String[] parts = details.split(", (?![^\\[]*\\])"); + for (int j = 0; j < parts.length; j++) { + boolean isLastDetail = (j == parts.length - 1); + String branchPrefix = isLastDetail ? "└── " : "├── "; + md.append(detailPrefix).append(branchPrefix).append(parts[j]).append("\n"); + } + } + } + + md.append("```\n\n"); + } + + private void renderGenericSection(StringBuilder md, String sectionName, List> rows) { + md.append("## ").append(sectionName).append("\n\n"); + + md.append("| Metric | Value | Details |\n"); + md.append("|--------|-------|----------|\n"); + + for (Map row : rows) { + String metric = (String) row.get("metric"); + String value = (String) row.get("value"); + String details = (String) row.get("details"); + + md.append("| ").append(metric).append(" | "); + md.append(value != null ? value : "").append(" | "); + md.append(details != null ? details : ""); + md.append(" |\n"); + } + + md.append("\n"); + } + + @Override + public String getName() { + return "summary-markdown"; + } + + @Override + public String getFileExtension() { + return "md"; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/TableFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/TableFormatter.java new file mode 100644 index 000000000..bab675354 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/TableFormatter.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import io.nosqlbench.nb.mql.query.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Formats query results as ASCII tables suitable for console output. + */ +public class TableFormatter implements ResultFormatter { + + private static final int DEFAULT_MAX_COLUMN_WIDTH = 50; + private static final String COLUMN_SEPARATOR = " | "; + private static final String HEADER_SEPARATOR_CHAR = "-"; + + @Override + public String format(QueryResult result) { + if (result.isEmpty()) { + return "No results found.\n"; + } + + List columns = result.columns(); + List> rows = result.rows(); + + // Calculate column widths + Map columnWidths = calculateColumnWidths(columns, rows); + + StringBuilder sb = new StringBuilder(); + + // Header + sb.append(formatRow(columns, columnWidths, columns)); + sb.append("\n"); + + // Separator + sb.append(formatSeparator(columns, columnWidths)); + sb.append("\n"); + + // Data rows + for (Map row : rows) { + List values = new ArrayList<>(); + for (String column : columns) { + Object value = row.get(column); + values.add(value != null ? value.toString() : "null"); + } + sb.append(formatRow(values, columnWidths, columns)); + sb.append("\n"); + } + + // Footer with row count + sb.append("\n"); + sb.append(result.rowCount()).append(" row"); + if (result.rowCount() != 1) { + sb.append("s"); + } + sb.append(" (").append(result.executionTimeMs()).append("ms)\n"); + + return sb.toString(); + } + + @Override + public String getName() { + return "table"; + } + + @Override + public String getFileExtension() { + return "txt"; + } + + private Map calculateColumnWidths(List columns, List> rows) { + Map widths = new java.util.LinkedHashMap<>(); + + // Initialize with header widths + for (String column : columns) { + widths.put(column, column.length()); + } + + // Check data widths + for (Map row : rows) { + for (String column : columns) { + Object value = row.get(column); + String strValue = value != null ? value.toString() : "null"; + int currentWidth = widths.get(column); + int valueWidth = Math.min(strValue.length(), DEFAULT_MAX_COLUMN_WIDTH); + widths.put(column, Math.max(currentWidth, valueWidth)); + } + } + + return widths; + } + + private String formatRow(List values, Map columnWidths, List columns) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columns.size(); i++) { + String column = columns.get(i); + String value = values.get(i); + int width = columnWidths.get(column); + + // Truncate if too long + if (value.length() > DEFAULT_MAX_COLUMN_WIDTH) { + value = value.substring(0, DEFAULT_MAX_COLUMN_WIDTH - 3) + "..."; + } + + // Pad to width + sb.append(padRight(value, width)); + + if (i < columns.size() - 1) { + sb.append(COLUMN_SEPARATOR); + } + } + return sb.toString(); + } + + private String formatSeparator(List columns, Map columnWidths) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columns.size(); i++) { + String column = columns.get(i); + int width = columnWidths.get(column); + sb.append(HEADER_SEPARATOR_CHAR.repeat(width)); + + if (i < columns.size() - 1) { + sb.append("-+-"); + } + } + return sb.toString(); + } + + private String padRight(String str, int width) { + if (str.length() >= width) { + return str; + } + return str + " ".repeat(width - str.length()); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/TsvFormatter.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/TsvFormatter.java new file mode 100644 index 000000000..e2c33327d --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/format/TsvFormatter.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.format; + +import io.nosqlbench.nb.mql.query.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Formats query results as TSV (Tab-Separated Values) for data processing tools. + * Compatible with awk, cut, paste, and other Unix text processing utilities. + */ +public class TsvFormatter implements ResultFormatter { + + private final boolean includeHeaders; + + public TsvFormatter() { + this(true); + } + + public TsvFormatter(boolean includeHeaders) { + this.includeHeaders = includeHeaders; + } + + @Override + public String format(QueryResult result) { + StringBuilder sb = new StringBuilder(); + + // Header row + if (includeHeaders) { + sb.append(String.join("\t", result.columns())); + sb.append("\n"); + } + + // Data rows + for (Map row : result.rows()) { + List values = new ArrayList<>(); + for (String column : result.columns()) { + Object value = row.get(column); + // TSV doesn't require quoting - just convert to string + // Tabs and newlines in data should be replaced + String strValue = value != null ? value.toString() : ""; + strValue = strValue.replace("\t", " ").replace("\n", " ").replace("\r", ""); + values.add(strValue); + } + sb.append(String.join("\t", values)); + sb.append("\n"); + } + + return sb.toString(); + } + + @Override + public String getName() { + return "tsv"; + } + + @Override + public String getFileExtension() { + return "tsv"; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/InvalidQueryException.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/InvalidQueryException.java new file mode 100644 index 000000000..dbcb8f512 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/InvalidQueryException.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.query; + +/** + * Exception thrown when query parameters are invalid. + * Should provide clear, actionable error messages to help users correct their queries. + */ +public class InvalidQueryException extends Exception { + + public InvalidQueryException(String message) { + super(message); + } + + public InvalidQueryException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/MetricsQueryCommand.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/MetricsQueryCommand.java new file mode 100644 index 000000000..6b284fe10 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/MetricsQueryCommand.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.query; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Map; + +/** + * Interface for all MetricsQL query commands. + * Each command translates high-level query parameters into SQL and executes against + * the NoSQLBench metrics database. + */ +public interface MetricsQueryCommand { + + /** + * Get the command name (e.g., "rate", "instant", "agg"). + */ + String getName(); + + /** + * Get a human-readable description of what this command does. + */ + String getDescription(); + + /** + * Execute the query command. + * + * @param conn Database connection + * @param params Query parameters (metric name, time windows, labels, etc.) + * @return Query result with columns and rows + * @throws SQLException If query execution fails + * @throws InvalidQueryException If parameters are invalid + */ + QueryResult execute(Connection conn, Map params) throws SQLException, InvalidQueryException; + + /** + * Validate query parameters before execution. + * Should throw InvalidQueryException with clear error messages if parameters are invalid. + * + * @param params Query parameters to validate + * @throws InvalidQueryException If parameters are invalid + */ + void validate(Map params) throws InvalidQueryException; + + /** + * Get usage examples for this command. + * Used for help text and documentation. + */ + default String getUsageExamples() { + return "No examples available."; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/QueryResult.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/QueryResult.java new file mode 100644 index 000000000..b77e5bd4e --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/QueryResult.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.query; + +import java.util.List; +import java.util.Map; + +/** + * Represents the result of a metrics query. + * + * @param columns List of column names in order + * @param rows List of row data as maps (column name -> value) + * @param executedSQL The SQL query that was executed (for debugging/explain) + * @param executionTimeMs Time taken to execute the query in milliseconds + */ +public record QueryResult( + List columns, + List> rows, + String executedSQL, + long executionTimeMs +) { + /** + * Get the number of rows returned. + */ + public int rowCount() { + return rows != null ? rows.size() : 0; + } + + /** + * Check if the result is empty. + */ + public boolean isEmpty() { + return rowCount() == 0; + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/TimeWindowParser.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/TimeWindowParser.java new file mode 100644 index 000000000..58f023cd8 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/query/TimeWindowParser.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.query; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses time window specifications like "5m", "1h", "30s" into milliseconds. + * Supports PromQL/MetricsQL time duration format. + */ +public class TimeWindowParser { + + // Pattern: number followed by time unit (s, m, h, d, w, y) + private static final Pattern TIME_WINDOW_PATTERN = Pattern.compile("^(\\d+(?:\\.\\d+)?)([smhdwy])$"); + + /** + * Parse a time window string into milliseconds. + * + * Supported formats: + * - "5s" = 5 seconds = 5000ms + * - "10m" = 10 minutes = 600000ms + * - "2h" = 2 hours = 7200000ms + * - "1d" = 1 day = 86400000ms + * - "1w" = 1 week = 604800000ms + * - "1y" = 1 year (365 days) = 31536000000ms + * + * @param window Time window string (e.g., "5m", "1h") + * @return Time window in milliseconds + * @throws InvalidQueryException If the format is invalid + */ + public static long parseToMillis(String window) throws InvalidQueryException { + if (window == null || window.trim().isEmpty()) { + throw new InvalidQueryException("Time window cannot be null or empty"); + } + + String normalized = window.trim().toLowerCase(); + Matcher matcher = TIME_WINDOW_PATTERN.matcher(normalized); + + if (!matcher.matches()) { + throw new InvalidQueryException( + "Invalid time window format: '" + window + "'\n" + + "Expected format: where unit is one of: s, m, h, d, w, y\n" + + "Examples: 5s, 10m, 2h, 1d" + ); + } + + double value; + try { + value = Double.parseDouble(matcher.group(1)); + } catch (NumberFormatException e) { + throw new InvalidQueryException("Invalid numeric value in time window: " + window, e); + } + + if (value <= 0) { + throw new InvalidQueryException("Time window must be positive, got: " + value); + } + + String unit = matcher.group(2); + long millisPerUnit = getMillisPerUnit(unit); + + // Check for overflow + double resultMs = value * millisPerUnit; + if (resultMs > Long.MAX_VALUE) { + throw new InvalidQueryException("Time window too large: " + window); + } + + return (long) resultMs; + } + + /** + * Parse a time window string, throwing RuntimeException on error. + * Useful for constants where you know the format is valid. + * + * @param window Time window string + * @return Time window in milliseconds + * @throws RuntimeException If parsing fails + */ + public static long parseToMillisUnchecked(String window) { + try { + return parseToMillis(window); + } catch (InvalidQueryException e) { + throw new RuntimeException("Failed to parse time window: " + window, e); + } + } + + /** + * Format milliseconds back to a human-readable time window string. + * Chooses the most appropriate unit automatically. + * + * @param millis Time in milliseconds + * @return Human-readable string (e.g., "5m", "2h") + */ + public static String formatMillis(long millis) { + if (millis == 0) { + return "0s"; + } + + // Try to find the largest unit that divides evenly + if (millis % MILLIS_PER_YEAR == 0) { + return (millis / MILLIS_PER_YEAR) + "y"; + } + if (millis % MILLIS_PER_WEEK == 0) { + return (millis / MILLIS_PER_WEEK) + "w"; + } + if (millis % MILLIS_PER_DAY == 0) { + return (millis / MILLIS_PER_DAY) + "d"; + } + if (millis % MILLIS_PER_HOUR == 0) { + return (millis / MILLIS_PER_HOUR) + "h"; + } + if (millis % MILLIS_PER_MINUTE == 0) { + return (millis / MILLIS_PER_MINUTE) + "m"; + } + if (millis % MILLIS_PER_SECOND == 0) { + return (millis / MILLIS_PER_SECOND) + "s"; + } + + // Fallback to milliseconds + return millis + "ms"; + } + + // Time unit constants + private static final long MILLIS_PER_SECOND = 1000L; + private static final long MILLIS_PER_MINUTE = 60L * MILLIS_PER_SECOND; + private static final long MILLIS_PER_HOUR = 60L * MILLIS_PER_MINUTE; + private static final long MILLIS_PER_DAY = 24L * MILLIS_PER_HOUR; + private static final long MILLIS_PER_WEEK = 7L * MILLIS_PER_DAY; + private static final long MILLIS_PER_YEAR = 365L * MILLIS_PER_DAY; // No leap years in PromQL + + private static long getMillisPerUnit(String unit) throws InvalidQueryException { + return switch (unit) { + case "s" -> MILLIS_PER_SECOND; + case "m" -> MILLIS_PER_MINUTE; + case "h" -> MILLIS_PER_HOUR; + case "d" -> MILLIS_PER_DAY; + case "w" -> MILLIS_PER_WEEK; + case "y" -> MILLIS_PER_YEAR; + default -> throw new InvalidQueryException("Unknown time unit: " + unit); + }; + } + + private TimeWindowParser() { + // Utility class - prevent instantiation + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/LabelSetResolver.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/LabelSetResolver.java new file mode 100644 index 000000000..46e4b8bd3 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/LabelSetResolver.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.schema; + +import java.util.Map; +import java.util.StringJoiner; + +/** + * Utility for computing label set hashes compatible with SqliteSnapshotReporter. + * This logic is intentionally duplicated from SqliteSnapshotReporter to maintain + * module independence (nb-api doesn't depend on nb-mql-api). + */ +public class LabelSetResolver { + + /** + * Compute a deterministic hash string for a label set. + * Format: {key1=value1,key2=value2,...} with keys sorted alphabetically. + * + * @param labels Map of label key-value pairs + * @return Hash string representation + */ + public static String computeHash(Map labels) { + if (labels == null || labels.isEmpty()) { + return "{}"; + } + StringJoiner joiner = new StringJoiner(",", "{", "}"); + labels.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> joiner.add(entry.getKey() + "=" + entry.getValue())); + return joiner.toString(); + } + + private LabelSetResolver() { + // Utility class - prevent instantiation + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/MetricsDatabaseReader.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/MetricsDatabaseReader.java new file mode 100644 index 000000000..182130aee --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/MetricsDatabaseReader.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.schema; + +import org.sqlite.SQLiteConfig; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +/** + * Manages read-only connections to NoSQLBench SQLite metrics databases. + * Supports querying both live sessions (via logs/metrics.db symlink) and + * historical session files. + */ +public class MetricsDatabaseReader { + + private static final Path DEFAULT_DB_PATH = Path.of("logs/metrics.db"); + private static final int DEFAULT_BUSY_TIMEOUT_MS = 5000; + + static { + // Explicitly load SQLite JDBC driver for standalone JAR usage + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("SQLite JDBC driver not found", e); + } + } + + /** + * Connect to the default metrics database (logs/metrics.db). + * This typically points to the current session via symlink. + * + * @return Read-only database connection + * @throws SQLException If connection fails or database doesn't exist + */ + public static Connection connect() throws SQLException { + return connect(DEFAULT_DB_PATH); + } + + /** + * Connect to a specific metrics database file. + * + * @param dbPath Path to the database file (can be a symlink) + * @return Read-only database connection + * @throws SQLException If connection fails or database doesn't exist + */ + public static Connection connect(Path dbPath) throws SQLException { + return connect(dbPath, DEFAULT_BUSY_TIMEOUT_MS); + } + + /** + * Connect to a specific metrics database with custom busy timeout. + * + * @param dbPath Path to the database file + * @param busyTimeoutMs Timeout in milliseconds for handling database locks + * @return Read-only database connection + * @throws SQLException If connection fails or database doesn't exist + */ + public static Connection connect(Path dbPath, int busyTimeoutMs) throws SQLException { + Path resolvedPath = (dbPath != null) ? dbPath : DEFAULT_DB_PATH; + + // Check if database file exists + if (!Files.exists(resolvedPath)) { + throw new SQLException( + "Database not found at: " + resolvedPath.toAbsolutePath() + "\n" + + "Hint: Ensure a NoSQLBench session is running or specify a session file with --db " + ); + } + + // Build JDBC URL + String jdbcUrl = "jdbc:sqlite:" + resolvedPath.toAbsolutePath(); + + // Configure SQLite for read-only access + SQLiteConfig config = new SQLiteConfig(); + config.setReadOnly(true); + config.setBusyTimeout(busyTimeoutMs); + + // Create connection with config + return DriverManager.getConnection(jdbcUrl, config.toProperties()); + } + + /** + * Connect to a database specified by JDBC URL string. + * Useful for testing with in-memory databases. + * + * @param jdbcUrl JDBC URL (e.g., "jdbc:sqlite::memory:") + * @return Read-only database connection + * @throws SQLException If connection fails + */ + public static Connection connectByUrl(String jdbcUrl) throws SQLException { + SQLiteConfig config = new SQLiteConfig(); + config.setReadOnly(true); + config.setBusyTimeout(DEFAULT_BUSY_TIMEOUT_MS); + + return DriverManager.getConnection(jdbcUrl, config.toProperties()); + } + + private MetricsDatabaseReader() { + // Utility class - prevent instantiation + } +} diff --git a/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/MetricsSchema.java b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/MetricsSchema.java new file mode 100644 index 000000000..df20950c9 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/java/io/nosqlbench/nb/mql/schema/MetricsSchema.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.schema; + +/** + * Schema constants for NoSQLBench SQLite metrics database. + * This schema is defined and created by SqliteSnapshotReporter in nb-api. + * These constants provide a read-only view for query operations. + */ +public class MetricsSchema { + + // Table names + public static final String TABLE_METRIC_FAMILY = "metric_family"; + public static final String TABLE_SAMPLE_NAME = "sample_name"; + public static final String TABLE_SAMPLE_VALUE = "sample_value"; + public static final String TABLE_SAMPLE_QUANTILE = "sample_quantile"; + public static final String TABLE_SAMPLE_RATE = "sample_rate"; + public static final String TABLE_SAMPLE_HISTOGRAM = "sample_histogram"; + public static final String TABLE_LABEL_KEY = "label_key"; + public static final String TABLE_LABEL_VALUE = "label_value"; + public static final String TABLE_LABEL_SET = "label_set"; + public static final String TABLE_LABEL_SET_MEMBERSHIP = "label_set_membership"; + + // metric_family columns + public static final String COL_MF_ID = "id"; + public static final String COL_MF_NAME = "name"; + public static final String COL_MF_HELP = "help"; + public static final String COL_MF_UNIT = "unit"; + public static final String COL_MF_TYPE = "type"; + + // sample_name columns + public static final String COL_SN_ID = "id"; + public static final String COL_SN_METRIC_FAMILY_ID = "metric_family_id"; + public static final String COL_SN_SAMPLE = "sample"; + + // sample_value columns + public static final String COL_SV_ID = "id"; + public static final String COL_SV_SAMPLE_NAME_ID = "sample_name_id"; + public static final String COL_SV_LABEL_SET_ID = "label_set_id"; + public static final String COL_SV_TIMESTAMP_MS = "timestamp_ms"; + public static final String COL_SV_VALUE = "value"; + public static final String COL_SV_COUNT = "count"; + public static final String COL_SV_SUM = "sum"; + public static final String COL_SV_MIN = "min"; + public static final String COL_SV_MAX = "max"; + public static final String COL_SV_MEAN = "mean"; + public static final String COL_SV_STDDEV = "stddev"; + + // sample_quantile columns + public static final String COL_SQ_SAMPLE_VALUE_ID = "sample_value_id"; + public static final String COL_SQ_QUANTILE = "quantile"; + public static final String COL_SQ_QUANTILE_VALUE = "quantile_value"; + + // sample_rate columns + public static final String COL_SR_SAMPLE_VALUE_ID = "sample_value_id"; + public static final String COL_SR_RATE_TYPE = "rate_type"; + public static final String COL_SR_RATE_VALUE = "rate_value"; + + // sample_histogram columns + public static final String COL_SH_SAMPLE_VALUE_ID = "sample_value_id"; + public static final String COL_SH_START_SECONDS = "start_seconds"; + public static final String COL_SH_INTERVAL_SECONDS = "interval_seconds"; + public static final String COL_SH_MAX_VALUE = "max_value"; + public static final String COL_SH_HISTOGRAM_BASE64 = "histogram_base64"; + + // label_key columns + public static final String COL_LK_ID = "id"; + public static final String COL_LK_NAME = "name"; + + // label_value columns + public static final String COL_LV_ID = "id"; + public static final String COL_LV_VALUE = "value"; + + // label_set columns + public static final String COL_LS_ID = "id"; + public static final String COL_LS_HASH = "hash"; + + // label_set_membership columns + public static final String COL_LSM_LABEL_SET_ID = "label_set_id"; + public static final String COL_LSM_LABEL_KEY_ID = "label_key_id"; + public static final String COL_LSM_LABEL_VALUE_ID = "label_value_id"; + + // Common SQL fragments for building queries + + /** + * SQL fragment to join sample_value with sample_name. + * Usage: FROM sample_value sv JOIN sample_name sn ON ... + */ + public static String joinSampleName() { + return String.format("%s.%s = %s.%s", + "sn", COL_SN_ID, + "sv", COL_SV_SAMPLE_NAME_ID); + } + + /** + * SQL fragment to join sample_value with label_set. + * Usage: FROM sample_value sv JOIN label_set ls ON ... + */ + public static String joinLabelSet() { + return String.format("%s.%s = %s.%s", + "ls", COL_LS_ID, + "sv", COL_SV_LABEL_SET_ID); + } + + /** + * SQL fragment to join label_set with label_set_membership. + * Usage: JOIN label_set_membership lsm ON ... + */ + public static String joinLabelSetMembership() { + return String.format("%s.%s = %s.%s", + "lsm", COL_LSM_LABEL_SET_ID, + "ls", COL_LS_ID); + } + + /** + * SQL fragment to join label_set_membership with label_key. + * Usage: JOIN label_key lk ON ... + */ + public static String joinLabelKey() { + return String.format("%s.%s = %s.%s", + "lk", COL_LK_ID, + "lsm", COL_LSM_LABEL_KEY_ID); + } + + /** + * SQL fragment to join label_set_membership with label_value. + * Usage: JOIN label_value lv ON ... + */ + public static String joinLabelValue() { + return String.format("%s.%s = %s.%s", + "lv", COL_LV_ID, + "lsm", COL_LSM_LABEL_VALUE_ID); + } + + /** + * Complete SQL fragment for joining all label-related tables. + * Produces: JOIN label_set ls ON ... LEFT JOIN label_set_membership lsm ON ... etc. + */ + public static String joinAllLabels() { + return String.format( + "JOIN %s ls ON %s " + + "LEFT JOIN %s lsm ON %s " + + "LEFT JOIN %s lk ON %s " + + "LEFT JOIN %s lv ON %s", + TABLE_LABEL_SET, joinLabelSet(), + TABLE_LABEL_SET_MEMBERSHIP, joinLabelSetMembership(), + TABLE_LABEL_KEY, joinLabelKey(), + TABLE_LABEL_VALUE, joinLabelValue() + ); + } + + private MetricsSchema() { + // Utility class - prevent instantiation + } +} diff --git a/nb-apis/nb-mql-api/src/main/resources/META-INF/services/io.nosqlbench.nb.api.apps.BundledApp b/nb-apis/nb-mql-api/src/main/resources/META-INF/services/io.nosqlbench.nb.api.apps.BundledApp new file mode 100644 index 000000000..a6685d71b --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/resources/META-INF/services/io.nosqlbench.nb.api.apps.BundledApp @@ -0,0 +1 @@ +io.nosqlbench.nb.mql.MQLApp diff --git a/nb-apis/nb-mql-api/src/main/resources/mql-examples.md b/nb-apis/nb-mql-api/src/main/resources/mql-examples.md new file mode 100644 index 000000000..cd8f01931 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/resources/mql-examples.md @@ -0,0 +1,124 @@ +# MetricsQL Query Examples + +This directory contains executable examples demonstrating all query commands in the MetricsQL query engine. Each example is validated by automated tests to ensure the documentation stays accurate. + +## Example Files by Theme + +### Basic Queries +- **[INSTANT_QUERIES.md](INSTANT_QUERIES.md)** - Get current/latest values + - Basic instant queries + - Label filtering + - Error metrics + - Multi-dimensional filtering + +- **[RANGE_QUERIES.md](RANGE_QUERIES.md)** - Time-series data retrieval + - Time-series over windows + - Historical data analysis + - Error tracking over time + +### Analysis Queries (Coming Soon) +- **RATE_QUERIES.md** - Calculate per-second rates + - Counter rate calculations + - Change detection + - Throughput analysis + +- **AGGREGATION_QUERIES.md** - Sum, average, min, max operations + - Grouping by labels + - Cross-dimensional aggregations + - Statistical summaries + +- **QUANTILE_QUERIES.md** - Percentile analysis + - Latency percentiles (p50, p95, p99) + - Timer statistics + - Performance analysis + +### Advanced Queries (Future) +- **RATIO_QUERIES.md** - Calculate error rates and ratios +- **TOPK_QUERIES.md** - Find top N values +- **BINARY_OPERATIONS.md** - Arithmetic between metrics + +## Test Database: examples.db + +All examples use `examples.db`, which contains metrics from a simplified e-commerce API: + +**Metrics:** +- `api_requests_total` - Request counters by method, endpoint, and status +- `api_latency_ms` - API latency timers with percentiles + +**Timeline:** +- 5 snapshots spanning 10 minutes (2025-10-23 10:00:00Z to 10:10:00Z) +- Simulates normal traffic → increased load → high load → recovery + +**Label Dimensions:** +- `method`: {GET, POST} +- `endpoint`: {/users, /products} +- `status`: {200, 404, 500} + +See [TEST_DATABASES.md](../TEST_DATABASES.md) for detailed schema information. + +## How Examples Work + +Each example follows this format: + +```markdown +## Example N: Description + +**Command:** +``` +command --param value --other param +``` + +**Expected Output:** +``` +[exact formatted table output] +``` + +**What it shows:** Explanation of the use case +``` + +The `ExampleDocumentationTest` class automatically: +1. Parses all example markdown files +2. Extracts commands and expected results +3. Executes each command against examples.db +4. Validates the actual output matches expectations + +**This means documentation is always accurate and tested!** + +## Running Examples Manually + +Once the CLI is implemented, you can run these examples: + +```bash +# Point to the examples database +export EXAMPLES_DB=src/test/resources/testdata/examples.db + +# Run any example command +nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total + +# With different output formats +nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total --format json +nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total --format csv +``` + +## Regenerating examples.db + +If you modify the schema or need to update the example data: + +```bash +mvn exec:java -pl nb-apis/nb-mql-api \ + -Dexec.mainClass="io.nosqlbench.nb.mql.testdata.ExampleDataGenerator" \ + -Dexec.classpathScope=test +``` + +## Contributing New Examples + +When adding a new query command: + +1. Add examples to the appropriate themed markdown file +2. Follow the established format with command, expected output, and explanation +3. Run the tests to verify: `mvn test -Dtest=ExampleDocumentationTest` +4. The tests will automatically validate your examples + +--- + +*These examples are automatically tested and validated with every build.* diff --git a/nb-apis/nb-mql-api/src/main/resources/mql-instant-queries.md b/nb-apis/nb-mql-api/src/main/resources/mql-instant-queries.md new file mode 100644 index 000000000..f49435898 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/resources/mql-instant-queries.md @@ -0,0 +1,130 @@ +# Instant Queries + +Get the most recent value for metrics. Instant queries return the latest snapshot for each unique label set. + +**Database:** `examples.db` + +--- + +## Example 1: Get All Latest Values + +**Command:** +``` +instant --metric api_requests_total +``` + +**Expected Output:** +``` +timestamp | value | labels +---------------------------------------------------------------------------------- +2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200 +2025-10-23 10:10:00.0 | 8500.0 | endpoint=/products, method=GET, status=200 +2025-10-23 10:10:00.0 | 1500.0 | endpoint=/users, method=POST, status=200 +2025-10-23 10:10:00.0 | 75.0 | endpoint=/users, method=GET, status=404 +2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500 + +5 rows (Xms) +``` + +**What it shows:** Retrieves the latest value for all label combinations of a metric. Useful for dashboards showing current state. + +**Use cases:** +- Current request counts by endpoint +- Latest error counts +- Current gauge values + +--- + +## Example 2: Filter by Specific Labels + +**Command:** +``` +instant --metric api_requests_total --labels method=GET,endpoint=/users,status=200 +``` + +**Expected Output:** +``` +timestamp | value | labels +---------------------------------------------------------------------------------- +2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200 + +1 row (Xms) +``` + +**What it shows:** Filtering to a specific label combination returns a single time series. + +**Use cases:** +- Monitor specific endpoint traffic +- Track particular operation types +- Isolate specific service behavior + +--- + +## Example 3: Query Only Errors + +**Command:** +``` +instant --metric api_requests_total --labels status=500 +``` + +**Expected Output:** +``` +timestamp | value | labels +-------------------------------------------------------------------------------- +2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500 + +1 row (Xms) +``` + +**What it shows:** Filter by a single label dimension (status code) to see current error count. + +**Use cases:** +- Current error count for alerting +- Quick health check +- SLA monitoring + +--- + +## Example 4: Filter Across Multiple Series + +**Command:** +``` +instant --metric api_requests_total --labels status=200 +``` + +**Expected Output:** +``` +timestamp | value | labels +---------------------------------------------------------------------------------- +2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200 +2025-10-23 10:10:00.0 | 8500.0 | endpoint=/products, method=GET, status=200 +2025-10-23 10:10:00.0 | 1500.0 | endpoint=/users, method=POST, status=200 + +3 rows (Xms) +``` + +**What it shows:** Filtering by a shared label value returns all matching label sets. This groups all successful requests while preserving endpoint/method distinctions. + +**Use cases:** +- All successful requests across endpoints +- All error types (status >= 400) +- Traffic patterns by method type + +--- + +## Key Concepts + +**Label Filtering:** +- Specify labels as comma-separated key=value pairs +- All specified labels must match (AND logic) +- Unspecified labels can have any value + +**Output:** +- `timestamp`: When the snapshot was captured +- `value`: The metric value at that time +- `labels`: All labels for this time series (comma-separated) + +**Performance:** +- Instant queries are fast (typically <5ms) +- Queries only the latest snapshot +- Efficient for real-time dashboards diff --git a/nb-apis/nb-mql-api/src/main/resources/mql-range-queries.md b/nb-apis/nb-mql-api/src/main/resources/mql-range-queries.md new file mode 100644 index 000000000..d7db3c470 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/resources/mql-range-queries.md @@ -0,0 +1,93 @@ +# Range Queries + +Retrieve time-series data over a time window. Range queries return all data points within a specified time range. + +**Database:** `examples.db` + +--- + +## Example 5: Time Series - Full History + +**Command:** +``` +range --metric api_requests_total --window 1h --labels method=GET,endpoint=/users,status=200 +``` + +**Expected Output:** +``` +timestamp | value | labels +---------------------------------------------------------------------------------- +2025-10-23 10:00:00.0 | 0.0 | endpoint=/users, method=GET, status=200 +2025-10-23 10:01:00.0 | 1000.0 | endpoint=/users, method=GET, status=200 +2025-10-23 10:02:00.0 | 2200.0 | endpoint=/users, method=GET, status=200 +2025-10-23 10:05:00.0 | 5500.0 | endpoint=/users, method=GET, status=200 +2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200 + +5 rows (Xms) +``` + +**What it shows:** Complete time-series progression showing how a counter grew from 0 to 11,000 requests over 10 minutes. + +**Use cases:** +- Visualize traffic patterns over time +- Analyze growth trends +- Export to graphing tools +- Historical analysis + +--- + +## Example 6: Track Errors Over Time + +**Command:** +``` +range --metric api_requests_total --window 1h --labels status=500 +``` + +**Expected Output:** +``` +timestamp | value | labels +-------------------------------------------------------------------------------- +2025-10-23 10:00:00.0 | 0.0 | endpoint=/users, method=GET, status=500 +2025-10-23 10:01:00.0 | 2.0 | endpoint=/users, method=GET, status=500 +2025-10-23 10:02:00.0 | 4.0 | endpoint=/users, method=GET, status=500 +2025-10-23 10:05:00.0 | 8.0 | endpoint=/users, method=GET, status=500 +2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500 + +5 rows (Xms) +``` + +**What it shows:** Track how server errors accumulated during the session. Notice the accelerated error rate during the high-load period (10:02 to 10:05). + +**Use cases:** +- Incident investigation +- Error rate trending +- Correlate errors with load spikes +- SLA violation detection + +--- + +## Key Concepts + +**Time Windows:** +- `--window 5m` - Last 5 minutes from latest snapshot +- `--window 1h` - Last hour +- `--last 10m` - Alternative syntax for --window + +**Alternative: Explicit Time Range (Future):** +``` +range --metric api_requests_total --start 1729681200000 --end 1729684800000 +``` + +**Output Ordering:** +- Results are ordered by timestamp (oldest first) +- Within same timestamp, ordered by labels alphabetically + +**Performance:** +- Range queries scan all data in the time window +- Use label filters to reduce result set +- Typical query time: <20ms for 1 hour of data + +**Visualization:** +- Export to CSV for Excel/Sheets: `--format csv` +- Export to JSON for plotting libraries: `--format json` +- Table format for quick console analysis diff --git a/nb-apis/nb-mql-api/src/main/resources/mql.md b/nb-apis/nb-mql-api/src/main/resources/mql.md new file mode 100644 index 000000000..85ad31f69 --- /dev/null +++ b/nb-apis/nb-mql-api/src/main/resources/mql.md @@ -0,0 +1,135 @@ +# MetricsQL Query Engine + +MetricsQL is a query engine for analyzing NoSQLBench SQLite metrics databases. It provides a MetricsQL/PromQL-inspired interface for querying time-series metrics data. + +## Quick Start + +```bash +# Show session information +nb5 mql session + +# List available metrics +nb5 mql summary + +# Get current metric values +nb5 mql instant --metric api_requests_total + +# Query time-series data +nb5 mql range --metric ops_total --window 5m + +# Calculate request rate +nb5 mql rate --metric requests_total --window 5m +``` + +## Available Commands + +### Discovery Commands +- **session** - Display session timeline and snapshot information +- **summary** - List all metrics with types and sample counts +- **sql** - Execute raw SQLite queries for advanced analysis + +### Query Commands +- **instant** - Get the most recent value for a metric +- **range** - Retrieve time-series data over a time window +- **topk** - Find top N label sets by metric value + +### Calculation Commands +- **rate** - Calculate per-second rate of change for counters +- **increase** - Calculate total increase in counter over time window +- **quantile** - Extract percentile values from timer/summary metrics +- **agg** - Perform aggregations (sum, avg, max, min, count) with grouping +- **ratio** - Calculate ratio between two metrics (e.g., error rate) + +## Output Formats + +All commands support multiple output formats via `--format`: + +- **table** (default) - ASCII tables for console +- **json** - Pretty-printed JSON for programmatic use +- **jsonl** - JSON Lines for streaming +- **csv** - Comma-separated values for spreadsheets +- **tsv** - Tab-separated values for Unix tools +- **markdown** - GitHub-flavored markdown tables + +## Examples + +See detailed examples organized by topic: +- [Examples Overview](mql-examples.md) - Navigation and all examples +- [Instant Queries](mql-instant-queries.md) - Point-in-time queries +- [Range Queries](mql-range-queries.md) - Time-series queries +- More examples coming for rate, aggregation, and quantile queries + +## Database Location + +By default, commands query `logs/metrics.db` which is a symlink to the current session's metrics database. + +To query a specific session: +```bash +nb5 mql instant --db logs/session_20251023_100000_metrics.db --metric ops_total +``` + +## Common Patterns + +### Monitoring +```bash +# Current error count +nb5 mql instant --metric api_requests_total --labels status=500 + +# Request rate last 5 minutes +nb5 mql rate --metric api_requests_total --window 5m --labels status=200 + +# p95 latency +nb5 mql quantile --metric api_latency --p 0.95 +``` + +### Analysis +```bash +# Total requests by endpoint +nb5 mql agg --metric api_requests_total --func sum --by endpoint + +# Top 10 busiest endpoints +nb5 mql topk --metric api_requests_total --n 10 + +# Error rate +nb5 mql ratio --numerator errors_total --denominator requests_total +``` + +### Exporting Data +```bash +# CSV for Excel +nb5 mql range --metric ops_total --window 1h --format csv > metrics.csv + +# JSON for analysis +nb5 mql summary --format json > metrics_inventory.json + +# Streaming JSONL +nb5 mql range --metric ops_total --window 1h --format jsonl | jq '.value' +``` + +## Advanced Usage + +### SQL Queries +```bash +# Custom aggregation +nb5 mql sql --query " + SELECT sn.sample, AVG(sv.value) as avg_value + FROM sample_value sv + JOIN sample_name sn ON sv.sample_name_id = sn.id + GROUP BY sn.sample +" + +# Schema inspection +nb5 mql sql --query "PRAGMA table_info(sample_value)" +``` + +### Debug Mode +```bash +# Show generated SQL +nb5 mql instant --metric ops_total --explain +``` + +## Learn More + +Run `nb5 help mql` to see this documentation. + +For detailed examples, see the [examples README](docs/mql/README.md). diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/AggregateCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/AggregateCommandTest.java new file mode 100644 index 000000000..5cf409831 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/AggregateCommandTest.java @@ -0,0 +1,229 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class AggregateCommandTest { + + @Test + void testSumNoGrouping() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "requests_total", + "function", "sum" + ); + QueryResult result = command.execute(conn, params); + + // Should return 1 row with total sum + assertEquals(1, result.rowCount()); + assertEquals(1, result.columns().size()); + assertEquals("sum", result.columns().get(0)); + + System.out.println("\n=== Aggregate: Sum (No Grouping) ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSumByActivity() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "requests_total", + "function", "sum", + "by", List.of("activity") + ); + QueryResult result = command.execute(conn, params); + + // Should return 2 rows (read and write) + assertEquals(2, result.rowCount(), "Should have 2 activities"); + assertEquals(2, result.columns().size()); + assertTrue(result.columns().contains("activity")); + assertTrue(result.columns().contains("sum")); + + System.out.println("\n=== Aggregate: Sum by Activity ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testAvgByRegion() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "requests_total", + "function", "avg", + "by", List.of("region") + ); + QueryResult result = command.execute(conn, params); + + // Should return 2 rows (us-east and us-west) + assertEquals(2, result.rowCount()); + assertTrue(result.columns().contains("region")); + assertTrue(result.columns().contains("avg")); + + System.out.println("\n=== Aggregate: Avg by Region ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testMaxByHost() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "requests_total", + "function", "max", + "by", List.of("host") + ); + QueryResult result = command.execute(conn, params); + + // Should return 3 rows (server1, server2, server3) + assertEquals(3, result.rowCount(), "Should have 3 hosts"); + + System.out.println("\n=== Aggregate: Max by Host ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testCountByMultipleLabels() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "requests_total", + "function", "count", + "by", List.of("activity", "region") + ); + QueryResult result = command.execute(conn, params); + + // Should return 4 rows (2 activities * 2 regions) + assertEquals(4, result.rowCount(), "Should have 4 combinations"); + assertEquals(3, result.columns().size()); + assertTrue(result.columns().contains("activity")); + assertTrue(result.columns().contains("region")); + assertTrue(result.columns().contains("count")); + + System.out.println("\n=== Aggregate: Count by Activity and Region ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSumWithLabelFilter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "requests_total", + "function", "sum", + "labels", Map.of("env", "prod") + ); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount()); + + System.out.println("\n=== Aggregate: Sum (env=prod only) ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSumExamplesDatabase() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + AggregateCommand command = new AggregateCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "function", "sum", + "by", List.of("status") + ); + QueryResult result = command.execute(conn, params); + + // Should return 3 rows (200, 404, 500) + assertEquals(3, result.rowCount()); + + System.out.println("\n=== Aggregate: Sum by Status Code ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testValidationMissingMetric() { + AggregateCommand command = new AggregateCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("function", "sum"))); + } + + @Test + void testValidationMissingFunction() { + AggregateCommand command = new AggregateCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test"))); + } + + @Test + void testValidationInvalidFunction() { + AggregateCommand command = new AggregateCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "function", "invalid"))); + } + + @Test + void testValidationValidFunctions() throws Exception { + AggregateCommand command = new AggregateCommand(); + + command.validate(Map.of("metric", "test", "function", "sum")); + command.validate(Map.of("metric", "test", "function", "avg")); + command.validate(Map.of("metric", "test", "function", "max")); + command.validate(Map.of("metric", "test", "function", "min")); + command.validate(Map.of("metric", "test", "function", "count")); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/IncreaseCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/IncreaseCommandTest.java new file mode 100644 index 000000000..a28adbcfc --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/IncreaseCommandTest.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class IncreaseCommandTest { + + @Test + void testIncreaseLinearPattern() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + IncreaseCommand command = new IncreaseCommand(); + + Map params = Map.of( + "metric", "patterns_total", + "window", "1h", + "labels", Map.of("pattern", "linear") + ); + QueryResult result = command.execute(conn, params); + + // Should return 1 row for the linear pattern label set + assertEquals(1, result.rowCount(), "Should have 1 result for linear pattern"); + + Map row = result.rows().get(0); + double increase = (double) row.get("increase"); + + // Linear pattern: 120 snapshots * 10 increment = 1190 total increase + assertEquals(1190.0, increase, 1.0, "Total increase should be ~1190"); + + System.out.println("\n=== Increase Query: Linear Pattern ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testIncreaseExponentialPattern() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + IncreaseCommand command = new IncreaseCommand(); + + Map params = Map.of( + "metric", "patterns_total", + "window", "1h", + "labels", Map.of("pattern", "exponential") + ); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount()); + + Map row = result.rows().get(0); + double increase = (double) row.get("increase"); + + // Exponential: 100 * (1.01^120) ≈ 230 + assertTrue(increase > 200, "Exponential increase should be > 200"); + assertTrue(increase < 300, "Exponential increase should be < 300"); + + System.out.println("\n=== Increase Query: Exponential Pattern ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testIncreaseStepPattern() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + IncreaseCommand command = new IncreaseCommand(); + + Map params = Map.of( + "metric", "patterns_total", + "window", "1h", + "labels", Map.of("pattern", "step") + ); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount()); + + Map row = result.rows().get(0); + double increase = (double) row.get("increase"); + + // Step pattern: starts at 100, ends at 1200 (12 steps * 100) + assertEquals(1100.0, increase, 1.0, "Step increase should be 1100"); + + System.out.println("\n=== Increase Query: Step Pattern ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testIncreaseExamplesDatabase() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + IncreaseCommand command = new IncreaseCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "window", "1h", + "labels", Map.of("method", "GET", "endpoint", "/users", "status", "200") + ); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount()); + + Map row = result.rows().get(0); + double increase = (double) row.get("increase"); + + // From 0 to 11000 + assertEquals(11000.0, increase, 0.1); + + System.out.println("\n=== Increase Query: Examples DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testIncreaseMultipleLabels() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + IncreaseCommand command = new IncreaseCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "window", "1h", + "labels", Map.of("status", "200") + ); + QueryResult result = command.execute(conn, params); + + // Should return 3 label sets (GET /users, GET /products, POST /users) + assertEquals(3, result.rowCount(), "Should have 3 label sets with status=200"); + + System.out.println("\n=== Increase Query: All Successful Requests ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testValidationMissingMetric() { + IncreaseCommand command = new IncreaseCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("window", "5m"))); + } + + @Test + void testValidationMissingWindow() { + IncreaseCommand command = new IncreaseCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test"))); + } + + @Test + void testValidationBothWindowAndLast() { + IncreaseCommand command = new IncreaseCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "window", "5m", "last", "10m"))); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/InstantCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/InstantCommandTest.java new file mode 100644 index 000000000..1fa901ba8 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/InstantCommandTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class InstantCommandTest { + + @Test + void testInstantQuerySimpleCounter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + InstantCommand command = new InstantCommand(); + + Map params = Map.of("metric", "activity_ops_total"); + QueryResult result = command.execute(conn, params); + + // Should return 2 rows (read and write activities) + assertEquals(2, result.rowCount(), "Expected 2 label sets"); + + // Check columns + assertEquals(3, result.columns().size()); + assertTrue(result.columns().contains("timestamp")); + assertTrue(result.columns().contains("value")); + assertTrue(result.columns().contains("labels")); + + // Verify we got both read and write + boolean hasRead = false; + boolean hasWrite = false; + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + double value = (double) row.get("value"); + + if (labels.contains("activity=read")) { + hasRead = true; + assertEquals(90.0, value, 0.01, "Expected final read value to be 90"); + } else if (labels.contains("activity=write")) { + hasWrite = true; + assertEquals(135.0, value, 0.01, "Expected final write value to be 135"); + } + } + + assertTrue(hasRead, "Should have read activity"); + assertTrue(hasWrite, "Should have write activity"); + + // Print formatted output for visual verification + System.out.println("\n=== Instant Query Result ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testInstantQueryWithLabelFilter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + InstantCommand command = new InstantCommand(); + + Map params = Map.of( + "metric", "activity_ops_total", + "labels", Map.of("activity", "read") + ); + QueryResult result = command.execute(conn, params); + + // Should return only 1 row (read activity) + assertEquals(1, result.rowCount(), "Expected 1 label set with activity=read filter"); + + Map row = result.rows().get(0); + String labels = (String) row.get("labels"); + double value = (double) row.get("value"); + + assertTrue(labels.contains("activity=read")); + assertEquals(90.0, value, 0.01); + } + } + + @Test + void testInstantQueryMultiDimensional() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + InstantCommand command = new InstantCommand(); + + Map params = Map.of("metric", "requests_total"); + QueryResult result = command.execute(conn, params); + + // Should return 24 rows (24 unique label combinations) + assertEquals(24, result.rowCount(), "Expected 24 unique label sets"); + + System.out.println("\n=== Multi-Dimensional Instant Query ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testInstantQueryWithMultipleLabelFilters() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + InstantCommand command = new InstantCommand(); + + Map params = Map.of( + "metric", "requests_total", + "labels", Map.of( + "activity", "write", + "region", "us-east", + "env", "prod" + ) + ); + QueryResult result = command.execute(conn, params); + + // Should return only rows matching all 3 label filters + // With 3 hosts, should get 3 results + assertEquals(3, result.rowCount(), + "Expected 3 label sets (one per host) matching activity=write, region=us-east, env=prod"); + + // Verify all results have the correct labels + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + assertTrue(labels.contains("activity=write")); + assertTrue(labels.contains("region=us-east")); + assertTrue(labels.contains("env=prod")); + } + } + } + + @Test + void testValidationMissingMetric() { + InstantCommand command = new InstantCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of())); + } + + @Test + void testValidationEmptyMetric() { + InstantCommand command = new InstantCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", ""))); + } + + @Test + void testValidationInvalidLabelsType() { + InstantCommand command = new InstantCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "labels", "not a map"))); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/QuantileCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/QuantileCommandTest.java new file mode 100644 index 000000000..6d2c1cb37 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/QuantileCommandTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class QuantileCommandTest { + + @Test + void testQuantileP95AllOperations() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + QuantileCommand command = new QuantileCommand(); + + Map params = Map.of( + "metric", "operation_latency", + "quantile", 0.95 + ); + QueryResult result = command.execute(conn, params); + + // Should have p95 values for all operations across all snapshots + assertTrue(result.rowCount() > 0, "Should have quantile results"); + + // Verify we have the expected columns + assertEquals(4, result.columns().size()); + assertTrue(result.columns().contains("timestamp")); + assertTrue(result.columns().contains("quantile")); + assertTrue(result.columns().contains("quantile_value")); + assertTrue(result.columns().contains("labels")); + + // All quantiles should be 0.95 + for (Map row : result.rows()) { + double q = (double) row.get("quantile"); + assertEquals(0.95, q, 0.001); + } + + System.out.println("\n=== Quantile Query: p95 All Operations ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testQuantileP99SpecificOperation() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + QuantileCommand command = new QuantileCommand(); + + Map params = Map.of( + "metric", "operation_latency", + "quantile", 0.99, + "labels", Map.of("operation", "select") + ); + QueryResult result = command.execute(conn, params); + + assertTrue(result.rowCount() > 0, "Should have p99 for select operation"); + + // All should be for select operation + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + assertTrue(labels.contains("operation=select")); + } + + System.out.println("\n=== Quantile Query: p99 Select Operation ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testQuantileP50Median() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + QuantileCommand command = new QuantileCommand(); + + Map params = Map.of( + "metric", "operation_latency", + "quantile", 0.50 + ); + QueryResult result = command.execute(conn, params); + + assertTrue(result.rowCount() > 0, "Should have median values"); + + System.out.println("\n=== Quantile Query: p50 (Median) ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testQuantileMultipleOperationsP95() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + QuantileCommand command = new QuantileCommand(); + + Map params = Map.of( + "metric", "operation_latency", + "quantile", 0.95 + ); + QueryResult result = command.execute(conn, params); + + // Should have results for all 4 operations (select, insert, update, delete) + // across 6 snapshots = 24 total + assertEquals(24, result.rowCount(), "Should have 24 quantile values (4 ops * 6 snapshots)"); + } + } + + @Test + void testValidationMissingMetric() { + QuantileCommand command = new QuantileCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("quantile", 0.95))); + } + + @Test + void testValidationMissingQuantile() { + QuantileCommand command = new QuantileCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test"))); + } + + @Test + void testValidationQuantileOutOfRange() { + QuantileCommand command = new QuantileCommand(); + + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "quantile", -0.1))); + + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "quantile", 1.5))); + } + + @Test + void testValidationValidQuantiles() throws Exception { + QuantileCommand command = new QuantileCommand(); + + command.validate(Map.of("metric", "test", "quantile", 0.0)); + command.validate(Map.of("metric", "test", "quantile", 0.5)); + command.validate(Map.of("metric", "test", "quantile", 1.0)); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RangeCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RangeCommandTest.java new file mode 100644 index 000000000..952fed69f --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RangeCommandTest.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class RangeCommandTest { + + @Test + void testRangeQueryAllData() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RangeCommand command = new RangeCommand(); + + // Query all data using a very large window + Map params = Map.of( + "metric", "activity_ops_total", + "window", "1h" + ); + QueryResult result = command.execute(conn, params); + + // Should return all 20 samples (10 snapshots × 2 label sets) + assertEquals(20, result.rowCount(), "Should return all samples"); + + // Verify columns + assertEquals(3, result.columns().size()); + assertTrue(result.columns().contains("timestamp")); + assertTrue(result.columns().contains("value")); + assertTrue(result.columns().contains("labels")); + + System.out.println("\n=== Range Query All Data ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testRangeQueryWithLabelFilter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RangeCommand command = new RangeCommand(); + + Map params = Map.of( + "metric", "activity_ops_total", + "window", "1h", + "labels", Map.of("activity", "read") + ); + QueryResult result = command.execute(conn, params); + + // Should return 10 samples (just read activity) + assertEquals(10, result.rowCount(), "Should return only read activity samples"); + + // Verify all have activity=read + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + assertTrue(labels.contains("activity=read"), "All results should be activity=read"); + assertFalse(labels.contains("activity=write"), "Should not contain write"); + } + + System.out.println("\n=== Range Query Filtered (read only) ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testRangeQueryVerifyTimeSeries() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RangeCommand command = new RangeCommand(); + + Map params = Map.of( + "metric", "activity_ops_total", + "window", "1h", + "labels", Map.of("activity", "read") + ); + QueryResult result = command.execute(conn, params); + + // Verify values form expected linear sequence: 0, 10, 20, ..., 90 + List values = result.rows().stream() + .map(row -> (Double) row.get("value")) + .sorted() + .toList(); + + assertEquals(10, values.size()); + for (int i = 0; i < 10; i++) { + assertEquals(i * 10.0, values.get(i), 0.01, + "Value at index " + i + " should be " + (i * 10)); + } + } + } + + @Test + void testRangeQueryExamplesDatabase() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RangeCommand command = new RangeCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "window", "1h", // Get all 5 snapshots + "labels", Map.of("method", "GET", "endpoint", "/users", "status", "200") + ); + QueryResult result = command.execute(conn, params); + + // Should return 5 snapshots + assertEquals(5, result.rowCount(), "Should return all 5 snapshots"); + + // Verify values are monotonically increasing: 0, 1000, 2200, 5500, 11000 + List values = result.rows().stream() + .map(row -> (Double) row.get("value")) + .toList(); + + assertEquals(0.0, values.get(0), 0.01); + assertEquals(1000.0, values.get(1), 0.01); + assertEquals(2200.0, values.get(2), 0.01); + assertEquals(5500.0, values.get(3), 0.01); + assertEquals(11000.0, values.get(4), 0.01); + + System.out.println("\n=== Range Query Examples DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testRangeQueryAlternativeLastSyntax() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RangeCommand command = new RangeCommand(); + + // Test --last as alternative to --window + Map params = Map.of( + "metric", "activity_ops_total", + "last", "1h" + ); + QueryResult result = command.execute(conn, params); + + assertEquals(20, result.rowCount(), "Should return all samples with --last"); + } + } + + @Test + void testValidationMissingMetric() { + RangeCommand command = new RangeCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("window", "5m"))); + } + + @Test + void testValidationMissingTimeRange() { + RangeCommand command = new RangeCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test"))); + } + + @Test + void testValidationMultipleTimeRangeMethods() { + RangeCommand command = new RangeCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "window", "5m", "last", "10m"))); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RateCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RateCommandTest.java new file mode 100644 index 000000000..a3758d287 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RateCommandTest.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class RateCommandTest { + + @Test + void testRateLinearPattern() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RateCommand command = new RateCommand(); + + Map params = Map.of( + "metric", "patterns_total", + "window", "1h", + "labels", Map.of("pattern", "linear") + ); + QueryResult result = command.execute(conn, params); + + // Linear pattern: increments by 10 every ~50ms + // Expected rate: 10 / 0.05 = 200 ops/sec (approximately) + assertTrue(result.rowCount() > 0, "Should have rate calculations"); + + // Get all calculated rates + List rates = result.rows().stream() + .map(row -> (Double) row.get("rate_per_sec")) + .filter(rate -> rate != null && !rate.isNaN()) + .toList(); + + assertTrue(rates.size() > 0, "Should have valid rates"); + + // For linear pattern, all rates should be relatively constant + // (within reasonable variance due to timing) + double avgRate = rates.stream().mapToDouble(Double::doubleValue).average().orElse(0); + assertTrue(avgRate > 0, "Average rate should be positive"); + + System.out.println("\n=== Rate Query: Linear Pattern ==="); + System.out.println(new TableFormatter().format(result)); + System.out.println("Average rate: " + avgRate + " ops/sec"); + } + } + + @Test + void testRateExponentialPattern() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RateCommand command = new RateCommand(); + + Map params = Map.of( + "metric", "patterns_total", + "window", "1h", + "labels", Map.of("pattern", "exponential") + ); + QueryResult result = command.execute(conn, params); + + assertTrue(result.rowCount() > 0, "Should have rate calculations"); + + // Exponential pattern: rate should increase over time + List rates = result.rows().stream() + .map(row -> (Double) row.get("rate_per_sec")) + .filter(rate -> rate != null && !rate.isNaN()) + .toList(); + + assertTrue(rates.size() > 10, "Should have many rate points"); + + // Verify rates are generally increasing (exponential growth) + // Compare first quarter to last quarter + int quarterSize = rates.size() / 4; + double firstQuarterAvg = rates.subList(0, quarterSize).stream() + .mapToDouble(Double::doubleValue).average().orElse(0); + double lastQuarterAvg = rates.subList(rates.size() - quarterSize, rates.size()).stream() + .mapToDouble(Double::doubleValue).average().orElse(0); + + assertTrue(lastQuarterAvg > firstQuarterAvg, + "Exponential pattern should have increasing rate over time"); + + System.out.println("\n=== Rate Query: Exponential Pattern ==="); + System.out.println("First quarter avg: " + firstQuarterAvg + " ops/sec"); + System.out.println("Last quarter avg: " + lastQuarterAvg + " ops/sec"); + } + } + + @Test + void testRateStepPattern() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RateCommand command = new RateCommand(); + + Map params = Map.of( + "metric", "patterns_total", + "window", "1h", + "labels", Map.of("pattern", "step") + ); + QueryResult result = command.execute(conn, params); + + assertTrue(result.rowCount() > 0, "Should have rate calculations"); + + // Step pattern: rate should be 0 within steps, high at step boundaries + List rates = result.rows().stream() + .map(row -> (Double) row.get("rate_per_sec")) + .filter(rate -> rate != null && !rate.isNaN()) + .toList(); + + // Most rates should be either 0 (flat) or very high (step jump) + long zeroRates = rates.stream().filter(r -> Math.abs(r) < 1.0).count(); + long highRates = rates.stream().filter(r -> r > 100.0).count(); + + assertTrue(zeroRates > 0 || highRates > 0, + "Step pattern should have either zero or high rates"); + + System.out.println("\n=== Rate Query: Step Pattern ==="); + System.out.println("Zero rates: " + zeroRates + ", High rates: " + highRates); + } + } + + @Test + void testRateExamplesDatabase() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + RateCommand command = new RateCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "window", "1h", + "labels", Map.of("method", "GET", "endpoint", "/users", "status", "200") + ); + QueryResult result = command.execute(conn, params); + + // Should have 4 rate calculations (5 snapshots - 1 for first point) + assertEquals(4, result.rowCount(), "Should have 4 rate points"); + + System.out.println("\n=== Rate Query: Examples DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testValidationMissingMetric() { + RateCommand command = new RateCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("window", "5m"))); + } + + @Test + void testValidationMissingWindow() { + RateCommand command = new RateCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test"))); + } + + @Test + void testValidationBothWindowAndLast() { + RateCommand command = new RateCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "window", "5m", "last", "10m"))); + } + + @Test + void testValidationWindowAllowed() throws Exception { + RateCommand command = new RateCommand(); + command.validate(Map.of("metric", "test", "window", "5m")); + } + + @Test + void testValidationLastAllowed() throws Exception { + RateCommand command = new RateCommand(); + command.validate(Map.of("metric", "test", "last", "5m")); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RatioCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RatioCommandTest.java new file mode 100644 index 000000000..42a1b27c8 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/RatioCommandTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class RatioCommandTest { + + @Test + void testRatioCommandExists() throws Exception { + // Basic smoke test - just verify command can be created and validated + RatioCommand command = new RatioCommand(); + assertEquals("ratio", command.getName()); + + // Valid params should pass validation + command.validate(Map.of( + "numerator", "metric1_total", + "denominator", "metric2_total" + )); + } + + + @Test + void testValidationMissingNumerator() { + RatioCommand command = new RatioCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("denominator", "total"))); + } + + @Test + void testValidationMissingDenominator() { + RatioCommand command = new RatioCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("numerator", "errors"))); + } + + @Test + void testValidationEmptyMetrics() { + RatioCommand command = new RatioCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("numerator", "", "denominator", "total"))); + } + + @Test + void testValidationValid() throws Exception { + RatioCommand command = new RatioCommand(); + command.validate(Map.of("numerator", "errors", "denominator", "total")); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SessionCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SessionCommandTest.java new file mode 100644 index 000000000..4eb7fd996 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SessionCommandTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class SessionCommandTest { + + @Test + void testSessionInfo() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SessionCommand command = new SessionCommand(); + + QueryResult result = command.execute(conn, Map.of()); + + // Should return 1 row with session info + assertEquals(1, result.rowCount()); + + // Verify columns + assertEquals(6, result.columns().size()); + assertTrue(result.columns().contains("first_snapshot")); + assertTrue(result.columns().contains("last_snapshot")); + assertTrue(result.columns().contains("duration")); + assertTrue(result.columns().contains("total_snapshots")); + assertTrue(result.columns().contains("total_samples")); + assertTrue(result.columns().contains("avg_interval")); + + Map row = result.rows().get(0); + + // Verify we have 5 snapshots + int snapshots = (int) row.get("total_snapshots"); + assertEquals(5, snapshots, "Should have 5 snapshots"); + + // Should have 35 total samples + int samples = (int) row.get("total_samples"); + assertEquals(35, samples, "Should have 35 samples"); + + System.out.println("\n=== Session Command ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSessionInfoSimpleCounter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SessionCommand command = new SessionCommand(); + + QueryResult result = command.execute(conn, Map.of()); + + assertEquals(1, result.rowCount()); + + Map row = result.rows().get(0); + int snapshots = (int) row.get("total_snapshots"); + int samples = (int) row.get("total_samples"); + + assertEquals(10, snapshots, "Should have 10 snapshots"); + assertEquals(20, samples, "Should have 20 samples"); + + System.out.println("\n=== Session Info: Simple Counter DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SqlCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SqlCommandTest.java new file mode 100644 index 000000000..0b9d2e8c7 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SqlCommandTest.java @@ -0,0 +1,206 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class SqlCommandTest { + + @Test + void testSqlQueryMetricFamilies() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SqlCommand command = new SqlCommand(); + + Map params = Map.of( + "query", "SELECT name, type FROM metric_family ORDER BY name" + ); + QueryResult result = command.execute(conn, params); + + // Should have at least 1 metric family + assertTrue(result.rowCount() >= 1, "Should have metric families"); + + // Verify columns + assertEquals(2, result.columns().size()); + assertTrue(result.columns().contains("name")); + assertTrue(result.columns().contains("type")); + + System.out.println("\n=== SQL Query: Metric Families ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSqlQuerySampleCounts() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SqlCommand command = new SqlCommand(); + + Map params = Map.of( + "query", "SELECT COUNT(*) as total_samples FROM sample_value" + ); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount()); + assertEquals(1, result.columns().size()); + assertEquals("total_samples", result.columns().get(0)); + + Map row = result.rows().get(0); + long count = ((Number) row.get("total_samples")).longValue(); + assertEquals(20, count, "Should have 20 samples"); + + System.out.println("\n=== SQL Query: Sample Count ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSqlQueryWithCTE() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SqlCommand command = new SqlCommand(); + + String sql = """ + WITH latest AS ( + SELECT MAX(timestamp_ms) as max_ts FROM sample_value + ) + SELECT COUNT(*) as latest_count + FROM sample_value sv, latest + WHERE sv.timestamp_ms = latest.max_ts + """; + + Map params = Map.of("query", sql); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount()); + long count = ((Number) result.rows().get(0).get("latest_count")).longValue(); + assertEquals(2, count, "Should have 2 samples at latest timestamp"); + } + } + + @Test + void testSqlQueryPragma() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SqlCommand command = new SqlCommand(); + + Map params = Map.of( + "query", "PRAGMA table_info(metric_family)" + ); + QueryResult result = command.execute(conn, params); + + // Should return schema information + assertTrue(result.rowCount() > 0, "Should return table info"); + + System.out.println("\n=== SQL Query: Table Schema ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testValidationMissingQuery() { + SqlCommand command = new SqlCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of())); + } + + @Test + void testValidationEmptyQuery() { + SqlCommand command = new SqlCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("query", ""))); + } + + @Test + void testValidationNonSelectQuery() { + SqlCommand command = new SqlCommand(); + + // INSERT should be blocked + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("query", "INSERT INTO metric_family ..."))); + + // UPDATE should be blocked + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("query", "UPDATE sample_value SET ..."))); + + // DELETE should be blocked + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("query", "DELETE FROM sample_value"))); + + // DROP should be blocked + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("query", "DROP TABLE metric_family"))); + } + + @Test + void testValidationAllowedQueries() throws Exception { + SqlCommand command = new SqlCommand(); + + // SELECT should be allowed + command.validate(Map.of("query", "SELECT * FROM metric_family")); + + // WITH (CTE) should be allowed + command.validate(Map.of("query", "WITH cte AS (SELECT 1) SELECT * FROM cte")); + + // PRAGMA should be allowed + command.validate(Map.of("query", "PRAGMA table_info(metric_family)")); + } + + @Test + void testSqlQueryExamples() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SqlCommand command = new SqlCommand(); + + // Example: Get all metrics with their latest values + String sql = """ + SELECT + sn.sample, + sv.value, + datetime(sv.timestamp_ms/1000, 'unixepoch') as captured_at + FROM sample_value sv + JOIN sample_name sn ON sv.sample_name_id = sn.id + WHERE sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) + ORDER BY sn.sample, sv.value DESC + """; + + Map params = Map.of("query", sql); + QueryResult result = command.execute(conn, params); + + assertTrue(result.rowCount() > 0, "Should return results"); + + System.out.println("\n=== SQL Query: All Latest Values ==="); + System.out.println(new TableFormatter().format(result)); + } + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SummaryCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SummaryCommandTest.java new file mode 100644 index 000000000..f28afcb17 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/SummaryCommandTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class SummaryCommandTest { + + @Test + void testSummaryExamplesDb() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SummaryCommand command = new SummaryCommand(); + + QueryResult result = command.execute(conn, Map.of()); + + // Should have multiple sections + assertTrue(result.rowCount() > 0, "Should have summary rows"); + + // Verify columns + assertEquals(4, result.columns().size()); + assertTrue(result.columns().contains("section")); + assertTrue(result.columns().contains("metric")); + assertTrue(result.columns().contains("value")); + assertTrue(result.columns().contains("details")); + + // Should have SESSION section + boolean hasSession = result.rows().stream() + .anyMatch(row -> "SESSION".equals(row.get("section"))); + assertTrue(hasSession, "Should have SESSION section"); + + System.out.println("\n=== Summary: Examples DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSummarySimpleCounter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SummaryCommand command = new SummaryCommand(); + + QueryResult result = command.execute(conn, Map.of()); + + assertTrue(result.rowCount() > 0, "Should have summary rows"); + + System.out.println("\n=== Summary: Simple Counter DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testSummaryMultiDimensional() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + SummaryCommand command = new SummaryCommand(); + + QueryResult result = command.execute(conn, Map.of()); + + assertTrue(result.rowCount() > 0, "Should have summary rows"); + + System.out.println("\n=== Summary: Multi-Dimensional DB ==="); + System.out.println(new TableFormatter().format(result)); + } + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/TopKCommandTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/TopKCommandTest.java new file mode 100644 index 000000000..150196c88 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/commands/TopKCommandTest.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.commands; + +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.InvalidQueryException; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class TopKCommandTest { + + @Test + void testTop3() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + TopKCommand command = new TopKCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "n", 3 + ); + QueryResult result = command.execute(conn, params); + + // Should return exactly 3 rows + assertEquals(3, result.rowCount(), "Should return top 3"); + + // Verify columns + assertEquals(2, result.columns().size()); + assertTrue(result.columns().contains("value")); + assertTrue(result.columns().contains("labels")); + + // First row should have highest value (11000) + double firstValue = (double) result.rows().get(0).get("value"); + assertEquals(11000.0, firstValue, 0.1, "Top value should be 11000"); + + System.out.println("\n=== TopK Query: Top 3 ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testTop1() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + TopKCommand command = new TopKCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "n", 1 + ); + QueryResult result = command.execute(conn, params); + + assertEquals(1, result.rowCount(), "Should return top 1"); + + double value = (double) result.rows().get(0).get("value"); + assertEquals(11000.0, value, 0.1); + + System.out.println("\n=== TopK Query: Top 1 ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testTop5WithLabelFilter() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase("examples.db"); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + TopKCommand command = new TopKCommand(); + + Map params = Map.of( + "metric", "api_requests_total", + "n", 5, + "labels", Map.of("status", "200") + ); + QueryResult result = command.execute(conn, params); + + // Should return 3 rows (only 3 label sets with status=200) + assertEquals(3, result.rowCount(), "Should return 3 (all status=200)"); + + // All should have status=200 + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + assertTrue(labels.contains("status=200")); + } + + System.out.println("\n=== TopK Query: Top 5 (filtered by status=200) ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testTop10MultiDimensional() throws Exception { + Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL); + + try (Connection conn = MetricsDatabaseReader.connect(dbPath)) { + TopKCommand command = new TopKCommand(); + + Map params = Map.of( + "metric", "requests_total", + "n", 10 + ); + QueryResult result = command.execute(conn, params); + + // Should return exactly 10 rows + assertEquals(10, result.rowCount(), "Should return top 10"); + + // Verify values are in descending order + double previousValue = Double.MAX_VALUE; + for (Map row : result.rows()) { + double value = (double) row.get("value"); + assertTrue(value <= previousValue, "Values should be in descending order"); + previousValue = value; + } + + System.out.println("\n=== TopK Query: Top 10 from Multi-Dimensional ==="); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testValidationMissingMetric() { + TopKCommand command = new TopKCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("n", 5))); + } + + @Test + void testValidationMissingN() { + TopKCommand command = new TopKCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test"))); + } + + @Test + void testValidationNegativeN() { + TopKCommand command = new TopKCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "n", -1))); + } + + @Test + void testValidationZeroN() { + TopKCommand command = new TopKCommand(); + assertThrows(InvalidQueryException.class, () -> + command.validate(Map.of("metric", "test", "n", 0))); + } + + @Test + void testValidationValidN() throws Exception { + TopKCommand command = new TopKCommand(); + command.validate(Map.of("metric", "test", "n", 1)); + command.validate(Map.of("metric", "test", "n", 100)); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/integration/ExampleDocumentationTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/integration/ExampleDocumentationTest.java new file mode 100644 index 000000000..414a3ba4c --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/integration/ExampleDocumentationTest.java @@ -0,0 +1,347 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.integration; + +import io.nosqlbench.nb.mql.commands.InstantCommand; +import io.nosqlbench.nb.mql.commands.RangeCommand; +import io.nosqlbench.nb.mql.format.TableFormatter; +import io.nosqlbench.nb.mql.query.QueryResult; +import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader; +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.Connection; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Validates that the examples in EXAMPLES.md produce the expected output. + * This test parses the markdown file, executes the commands, and verifies + * the results match the documented expectations. + */ +class ExampleDocumentationTest { + + private static Path examplesDb; + private static List examples; + + @BeforeAll + static void loadExamples() throws IOException { + // Load the examples database + examplesDb = TestDatabaseLoader.getDatabase("examples.db"); + + // Parse all example markdown files from classpath + examples = new ArrayList<>(); + + String[] exampleFiles = { + "mql-instant-queries.md", + "mql-range-queries.md" + // Add more as they are created: + // "mql-rate-queries.md", + // "mql-aggregation-queries.md", + // etc. + }; + + for (String exampleFile : exampleFiles) { + String content = new String( + ExampleDocumentationTest.class.getClassLoader() + .getResourceAsStream(exampleFile) + .readAllBytes() + ); + examples.addAll(parseExamples(content)); + System.out.println("Loaded examples from " + exampleFile); + } + + System.out.println("Total: " + examples.size() + " examples loaded"); + } + + @Test + void testExample1_InstantQueryAllLabels() throws Exception { + Example example = findExample("Example 1"); + assertNotNull(example, "Example 1 should exist"); + + try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) { + QueryResult result = executeCommand(conn, example.command); + + // Verify row count + assertEquals(5, result.rowCount(), "Example 1 should return 5 label sets"); + + // Verify all values are present in the result + Map expectedValues = Map.of( + "endpoint=/users, method=GET, status=200", 11000.0, + "endpoint=/products, method=GET, status=200", 8500.0, + "endpoint=/users, method=POST, status=200", 1500.0, + "endpoint=/users, method=GET, status=404", 75.0, + "endpoint=/users, method=GET, status=500", 15.0 + ); + + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + double value = (double) row.get("value"); + + boolean matched = false; + for (Map.Entry expected : expectedValues.entrySet()) { + if (labelsMatch(labels, expected.getKey())) { + assertEquals(expected.getValue(), value, 0.01, + "Value mismatch for labels: " + labels); + matched = true; + break; + } + } + assertTrue(matched, "Unexpected label set: " + labels); + } + + System.out.println("Example 1 output:"); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testExample2_InstantQueryWithFilter() throws Exception { + Example example = findExample("Example 2"); + assertNotNull(example, "Example 2 should exist"); + + try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) { + QueryResult result = executeCommand(conn, example.command); + + // Should return exactly 1 row + assertEquals(1, result.rowCount(), "Filtered query should return 1 row"); + + Map row = result.rows().get(0); + double value = (double) row.get("value"); + String labels = (String) row.get("labels"); + + assertEquals(11000.0, value, 0.01); + assertTrue(labels.contains("method=GET")); + assertTrue(labels.contains("endpoint=/users")); + assertTrue(labels.contains("status=200")); + + System.out.println("Example 2 output:"); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testExample3_InstantQueryErrors() throws Exception { + Example example = findExample("Example 3"); + assertNotNull(example, "Example 3 should exist"); + + try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) { + QueryResult result = executeCommand(conn, example.command); + + assertEquals(1, result.rowCount(), "Should return 1 error metric"); + + Map row = result.rows().get(0); + double value = (double) row.get("value"); + String labels = (String) row.get("labels"); + + assertEquals(15.0, value, 0.01); + assertTrue(labels.contains("status=500")); + + System.out.println("Example 3 output:"); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testExample4_InstantQuerySuccessful() throws Exception { + Example example = findExample("Example 4"); + assertNotNull(example, "Example 4 should exist"); + + try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) { + QueryResult result = executeCommand(conn, example.command); + + assertEquals(3, result.rowCount(), "Should return 3 successful request types"); + + // Verify all have status=200 + for (Map row : result.rows()) { + String labels = (String) row.get("labels"); + assertTrue(labels.contains("status=200"), "All results should have status=200"); + } + + System.out.println("Example 4 output:"); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testExample5_RangeQueryTimeSeries() throws Exception { + Example example = findExample("Example 5"); + assertNotNull(example, "Example 5 should exist"); + + try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) { + QueryResult result = executeCommand(conn, example.command); + + // Should return 5 snapshots for the single label set + assertEquals(5, result.rowCount(), "Should return 5 time-series points"); + + // Verify values are: 0, 1000, 2200, 5500, 11000 + List values = result.rows().stream() + .map(row -> (Double) row.get("value")) + .toList(); + + assertEquals(0.0, values.get(0), 0.01); + assertEquals(1000.0, values.get(1), 0.01); + assertEquals(2200.0, values.get(2), 0.01); + assertEquals(5500.0, values.get(3), 0.01); + assertEquals(11000.0, values.get(4), 0.01); + + System.out.println("Example 5 output:"); + System.out.println(new TableFormatter().format(result)); + } + } + + @Test + void testExample6_RangeQueryErrors() throws Exception { + Example example = findExample("Example 6"); + assertNotNull(example, "Example 6 should exist"); + + try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) { + QueryResult result = executeCommand(conn, example.command); + + // Should return 5 snapshots for error metric + assertEquals(5, result.rowCount(), "Should return 5 time-series points for errors"); + + // Verify error values: 0, 2, 4, 8, 15 + List values = result.rows().stream() + .map(row -> (Double) row.get("value")) + .toList(); + + assertEquals(0.0, values.get(0), 0.01); + assertEquals(2.0, values.get(1), 0.01); + assertEquals(4.0, values.get(2), 0.01); + assertEquals(8.0, values.get(3), 0.01); + assertEquals(15.0, values.get(4), 0.01); + + System.out.println("Example 6 output:"); + System.out.println(new TableFormatter().format(result)); + } + } + + // Helper methods + + private static QueryResult executeCommand(Connection conn, String commandStr) throws Exception { + // Parse command string: "instant --metric [--labels key=val,...]" + Map params = parseCommandParams(commandStr); + + if (commandStr.startsWith("instant")) { + InstantCommand command = new InstantCommand(); + return command.execute(conn, params); + } else if (commandStr.startsWith("range")) { + RangeCommand command = new RangeCommand(); + return command.execute(conn, params); + } + + throw new UnsupportedOperationException("Command not yet implemented: " + commandStr); + } + + private static Map parseCommandParams(String commandStr) { + Map params = new LinkedHashMap<>(); + + // Extract --metric value + Pattern metricPattern = Pattern.compile("--metric\\s+(\\S+)"); + Matcher metricMatcher = metricPattern.matcher(commandStr); + if (metricMatcher.find()) { + params.put("metric", metricMatcher.group(1)); + } + + // Extract --window value + Pattern windowPattern = Pattern.compile("--window\\s+(\\S+)"); + Matcher windowMatcher = windowPattern.matcher(commandStr); + if (windowMatcher.find()) { + params.put("window", windowMatcher.group(1)); + } + + // Extract --last value + Pattern lastPattern = Pattern.compile("--last\\s+(\\S+)"); + Matcher lastMatcher = lastPattern.matcher(commandStr); + if (lastMatcher.find()) { + params.put("last", lastMatcher.group(1)); + } + + // Extract --labels key=val,key=val + Pattern labelsPattern = Pattern.compile("--labels\\s+(\\S+)"); + Matcher labelsMatcher = labelsPattern.matcher(commandStr); + if (labelsMatcher.find()) { + String labelsStr = labelsMatcher.group(1); + Map labels = new LinkedHashMap<>(); + for (String pair : labelsStr.split(",")) { + String[] kv = pair.split("=", 2); + if (kv.length == 2) { + labels.put(kv[0], kv[1]); + } + } + params.put("labels", labels); + } + + return params; + } + + private static boolean labelsMatch(String actual, String expected) { + // Normalize: remove spaces, sort key=value pairs + String[] actualPairs = actual.replace(" ", "").split(","); + String[] expectedPairs = expected.replace(" ", "").split(","); + + if (actualPairs.length != expectedPairs.length) { + return false; + } + + java.util.Arrays.sort(actualPairs); + java.util.Arrays.sort(expectedPairs); + + return java.util.Arrays.equals(actualPairs, expectedPairs); + } + + private static Example findExample(String exampleName) { + for (Example example : examples) { + if (example.name.contains(exampleName)) { + return example; + } + } + return null; + } + + private static List parseExamples(String markdown) { + List result = new ArrayList<>(); + + // Pattern to match example sections + Pattern examplePattern = Pattern.compile( + "##\\s+(Example \\d+:[^\\n]+).*?" + + "\\*\\*Command:\\*\\*\\s*```([^`]+)```", + Pattern.DOTALL + ); + + Matcher matcher = examplePattern.matcher(markdown); + while (matcher.find()) { + String name = matcher.group(1).trim(); + String command = matcher.group(2).trim(); + result.add(new Example(name, command)); + } + + return result; + } + + private record Example(String name, String command) {} +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/query/TimeWindowParserTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/query/TimeWindowParserTest.java new file mode 100644 index 000000000..fc8f90a80 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/query/TimeWindowParserTest.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.query; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class TimeWindowParserTest { + + @Test + void testParseSeconds() throws InvalidQueryException { + assertEquals(5000L, TimeWindowParser.parseToMillis("5s")); + assertEquals(30000L, TimeWindowParser.parseToMillis("30s")); + assertEquals(1000L, TimeWindowParser.parseToMillis("1s")); + } + + @Test + void testParseMinutes() throws InvalidQueryException { + assertEquals(60000L, TimeWindowParser.parseToMillis("1m")); + assertEquals(300000L, TimeWindowParser.parseToMillis("5m")); + assertEquals(900000L, TimeWindowParser.parseToMillis("15m")); + } + + @Test + void testParseHours() throws InvalidQueryException { + assertEquals(3600000L, TimeWindowParser.parseToMillis("1h")); + assertEquals(7200000L, TimeWindowParser.parseToMillis("2h")); + assertEquals(86400000L, TimeWindowParser.parseToMillis("24h")); + } + + @Test + void testParseDays() throws InvalidQueryException { + assertEquals(86400000L, TimeWindowParser.parseToMillis("1d")); + assertEquals(604800000L, TimeWindowParser.parseToMillis("7d")); + } + + @Test + void testParseWeeks() throws InvalidQueryException { + assertEquals(604800000L, TimeWindowParser.parseToMillis("1w")); + assertEquals(1209600000L, TimeWindowParser.parseToMillis("2w")); + } + + @Test + void testParseYears() throws InvalidQueryException { + assertEquals(31536000000L, TimeWindowParser.parseToMillis("1y")); + } + + @Test + void testParseDecimal() throws InvalidQueryException { + assertEquals(1500L, TimeWindowParser.parseToMillis("1.5s")); + assertEquals(90000L, TimeWindowParser.parseToMillis("1.5m")); + } + + @Test + void testCaseInsensitive() throws InvalidQueryException { + assertEquals(5000L, TimeWindowParser.parseToMillis("5S")); + assertEquals(300000L, TimeWindowParser.parseToMillis("5M")); + assertEquals(3600000L, TimeWindowParser.parseToMillis("1H")); + } + + @Test + void testWhitespace() throws InvalidQueryException { + assertEquals(5000L, TimeWindowParser.parseToMillis(" 5s ")); + assertEquals(300000L, TimeWindowParser.parseToMillis("\t5m\n")); + } + + @Test + void testInvalidFormat() { + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("5")); + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("s")); + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("5x")); + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("five seconds")); + } + + @Test + void testNullOrEmpty() { + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis(null)); + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("")); + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis(" ")); + } + + @Test + void testNegativeValue() { + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("-5s")); + } + + @Test + void testZeroValue() { + assertThrows(InvalidQueryException.class, () -> + TimeWindowParser.parseToMillis("0s")); + } + + @Test + void testFormatMillis() { + assertEquals("5s", TimeWindowParser.formatMillis(5000L)); + assertEquals("5m", TimeWindowParser.formatMillis(300000L)); + assertEquals("1h", TimeWindowParser.formatMillis(3600000L)); + assertEquals("1d", TimeWindowParser.formatMillis(86400000L)); + assertEquals("1w", TimeWindowParser.formatMillis(604800000L)); + } + + @Test + void testFormatMillisNonExact() { + // Should fall back to smallest unit that works + assertEquals("1500ms", TimeWindowParser.formatMillis(1500L)); + assertEquals("90s", TimeWindowParser.formatMillis(90000L)); + } + + @Test + void testRoundTrip() throws InvalidQueryException { + String[] inputs = {"5s", "10m", "2h", "1d", "1w"}; + for (String input : inputs) { + long millis = TimeWindowParser.parseToMillis(input); + String formatted = TimeWindowParser.formatMillis(millis); + assertEquals(input, formatted, "Round trip failed for: " + input); + } + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/schema/LabelSetResolverTest.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/schema/LabelSetResolverTest.java new file mode 100644 index 000000000..4e3342e83 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/schema/LabelSetResolverTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.schema; + +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class LabelSetResolverTest { + + @Test + void testEmptyLabels() { + assertEquals("{}", LabelSetResolver.computeHash(Map.of())); + assertEquals("{}", LabelSetResolver.computeHash(null)); + } + + @Test + void testSingleLabel() { + Map labels = Map.of("activity", "write"); + assertEquals("{activity=write}", LabelSetResolver.computeHash(labels)); + } + + @Test + void testMultipleLabels() { + Map labels = Map.of( + "activity", "write", + "host", "server1", + "region", "us-east" + ); + // Should be sorted alphabetically by key + assertEquals("{activity=write,host=server1,region=us-east}", + LabelSetResolver.computeHash(labels)); + } + + @Test + void testLabelsSortedAlphabetically() { + // Insert in reverse order to test sorting + Map labels = new LinkedHashMap<>(); + labels.put("z_last", "value3"); + labels.put("b_second", "value2"); + labels.put("a_first", "value1"); + + String hash = LabelSetResolver.computeHash(labels); + assertEquals("{a_first=value1,b_second=value2,z_last=value3}", hash); + } + + @Test + void testSameLabelsDifferentOrder() { + Map labels1 = new LinkedHashMap<>(); + labels1.put("activity", "write"); + labels1.put("host", "server1"); + + Map labels2 = new LinkedHashMap<>(); + labels2.put("host", "server1"); + labels2.put("activity", "write"); + + // Should produce same hash regardless of insertion order + assertEquals(LabelSetResolver.computeHash(labels1), + LabelSetResolver.computeHash(labels2)); + } + + @Test + void testSpecialCharacters() { + Map labels = Map.of( + "label", "value=with=equals", + "another", "value,with,commas" + ); + String hash = LabelSetResolver.computeHash(labels); + assertTrue(hash.contains("label=value=with=equals")); + assertTrue(hash.contains("another=value,with,commas")); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/ExampleDataGenerator.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/ExampleDataGenerator.java new file mode 100644 index 000000000..bf6ba8896 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/ExampleDataGenerator.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.testdata; + +import java.nio.file.Path; + +/** + * Generates the reference example database used in EXAMPLES.md documentation. + * This database has specific, well-known timestamps and values that are referenced + * in documentation and validated by ExampleDocumentationTest. + * + * The data represents a simplified e-commerce API with: + * - Request counters (GET/POST operations on different endpoints) + * - API latency timers + * - HTTP status codes (200, 404, 500) + * + * Timeline: + * - 2025-10-23 10:00:00Z - T+0: Initial state (zeros) + * - 2025-10-23 10:01:00Z - T+1min: Normal traffic + * - 2025-10-23 10:02:00Z - T+2min: Increased load + * - 2025-10-23 10:05:00Z - T+5min: High load period + * - 2025-10-23 10:10:00Z - T+10min: Back to normal + */ +public class ExampleDataGenerator { + + /** + * Generate examples.db - the reference database for documentation examples. + */ + public static void generateExamples(Path outputPath) throws Exception { + MetricsDataBuilder.create(outputPath) + // T+0: Initial state (10:00:00) + .atTime("2025-10-23T10:00:00Z") + .counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "200") + .counter("api_requests", 0, "method", "GET", "endpoint", "/products", "status", "200") + .counter("api_requests", 0, "method", "POST", "endpoint", "/users", "status", "200") + .counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "404") + .counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "500") + .timer("api_latency", new long[]{10, 12, 15, 20, 25}, "method", "GET", "endpoint", "/users") + .timer("api_latency", new long[]{8, 10, 12, 14, 16}, "method", "GET", "endpoint", "/products") + + // T+1min: Normal traffic (10:01:00) + .atTime("2025-10-23T10:01:00Z") + .counter("api_requests", 1000, "method", "GET", "endpoint", "/users", "status", "200") + .counter("api_requests", 800, "method", "GET", "endpoint", "/products", "status", "200") + .counter("api_requests", 150, "method", "POST", "endpoint", "/users", "status", "200") + .counter("api_requests", 5, "method", "GET", "endpoint", "/users", "status", "404") + .counter("api_requests", 2, "method", "GET", "endpoint", "/users", "status", "500") + .timer("api_latency", new long[]{10, 12, 15, 18, 22, 25, 30}, "method", "GET", "endpoint", "/users") + .timer("api_latency", new long[]{8, 10, 11, 13, 15, 18}, "method", "GET", "endpoint", "/products") + + // T+2min: Increased load (10:02:00) + .atTime("2025-10-23T10:02:00Z") + .counter("api_requests", 2200, "method", "GET", "endpoint", "/users", "status", "200") + .counter("api_requests", 1650, "method", "GET", "endpoint", "/products", "status", "200") + .counter("api_requests", 320, "method", "POST", "endpoint", "/users", "status", "200") + .counter("api_requests", 12, "method", "GET", "endpoint", "/users", "status", "404") + .counter("api_requests", 4, "method", "GET", "endpoint", "/users", "status", "500") + .timer("api_latency", new long[]{12, 15, 18, 22, 28, 35, 42}, "method", "GET", "endpoint", "/users") + .timer("api_latency", new long[]{9, 11, 13, 15, 18, 22}, "method", "GET", "endpoint", "/products") + + // T+5min: High load period (10:05:00) + .atTime("2025-10-23T10:05:00Z") + .counter("api_requests", 5500, "method", "GET", "endpoint", "/users", "status", "200") + .counter("api_requests", 4200, "method", "GET", "endpoint", "/products", "status", "200") + .counter("api_requests", 750, "method", "POST", "endpoint", "/users", "status", "200") + .counter("api_requests", 35, "method", "GET", "endpoint", "/users", "status", "404") + .counter("api_requests", 8, "method", "GET", "endpoint", "/users", "status", "500") + .timer("api_latency", new long[]{15, 20, 25, 32, 45, 58, 75}, "method", "GET", "endpoint", "/users") + .timer("api_latency", new long[]{10, 13, 16, 20, 25, 30}, "method", "GET", "endpoint", "/products") + + // T+10min: Back to normal (10:10:00) + .atTime("2025-10-23T10:10:00Z") + .counter("api_requests", 11000, "method", "GET", "endpoint", "/users", "status", "200") + .counter("api_requests", 8500, "method", "GET", "endpoint", "/products", "status", "200") + .counter("api_requests", 1500, "method", "POST", "endpoint", "/users", "status", "200") + .counter("api_requests", 75, "method", "GET", "endpoint", "/users", "status", "404") + .counter("api_requests", 15, "method", "GET", "endpoint", "/users", "status", "500") + .timer("api_latency", new long[]{11, 14, 17, 22, 28, 35, 45}, "method", "GET", "endpoint", "/users") + .timer("api_latency", new long[]{9, 11, 13, 16, 19, 24}, "method", "GET", "endpoint", "/products") + + .build(); + } + + /** + * Main method to generate the examples database. + */ + public static void main(String[] args) throws Exception { + Path examplesDb = Path.of("nb-apis/nb-mql-api/src/test/resources/testdata/examples.db"); + + System.out.println("Generating examples.db for documentation..."); + generateExamples(examplesDb); + System.out.println("✓ Generated examples.db"); + System.out.println("Location: " + examplesDb.toAbsolutePath()); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/MetricsDataBuilder.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/MetricsDataBuilder.java new file mode 100644 index 000000000..3f2d5a244 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/MetricsDataBuilder.java @@ -0,0 +1,249 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.testdata; + +import io.nosqlbench.nb.api.components.core.NBBaseComponent; +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir; +import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory; +import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter; +import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer; +import io.nosqlbench.nb.api.engine.metrics.reporters.MetricInstanceFilter; +import io.nosqlbench.nb.api.engine.metrics.reporters.SqliteSnapshotReporter; +import io.nosqlbench.nb.api.engine.metrics.view.MetricsView; +import io.nosqlbench.nb.api.labels.NBLabels; + +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Simplified utility for creating test metrics databases with specific timestamps. + * Provides a fluent API for building predictable test data for documentation examples. + * + * Example: + * 
+ * MetricsDataBuilder.create("examples.db")
+ *     .atTime("2025-10-23T10:00:00Z")
+ *         .counter("ops_total", 100, "activity", "read")
+ *         .counter("ops_total", 150, "activity", "write")
+ *     .atTime("2025-10-23T10:01:00Z")
+ *         .counter("ops_total", 120, "activity", "read")
+ *         .counter("ops_total", 180, "activity", "write")
+ *     .build();
+ *
+ */ +public class MetricsDataBuilder { + + private final Path outputPath; + private final List snapshots = new ArrayList<>(); + private Snapshot currentSnapshot; + + private MetricsDataBuilder(Path outputPath) { + this.outputPath = outputPath; + } + + /** + * Create a new metrics data builder for the specified database file. + * + * @param dbPath Path to the output database file + * @return New builder instance + */ + public static MetricsDataBuilder create(String dbPath) { + return new MetricsDataBuilder(Path.of(dbPath)); + } + + /** + * Create a new metrics data builder for the specified path. + * + * @param outputPath Path to the output database file + * @return New builder instance + */ + public static MetricsDataBuilder create(Path outputPath) { + return new MetricsDataBuilder(outputPath); + } + + /** + * Start a new snapshot at the specified timestamp (ISO-8601 format). + * + * @param timestamp ISO-8601 timestamp (e.g., "2025-10-23T10:00:00Z") + * @return This builder for chaining + */ + public MetricsDataBuilder atTime(String timestamp) { + Instant instant = Instant.parse(timestamp); + return atTime(instant); + } + + /** + * Start a new snapshot at the specified timestamp. + * + * @param timestamp Instant timestamp + * @return This builder for chaining + */ + public MetricsDataBuilder atTime(Instant timestamp) { + if (currentSnapshot != null) { + snapshots.add(currentSnapshot); + } + currentSnapshot = new Snapshot(timestamp); + return this; + } + + /** + * Add a counter metric to the current snapshot. + * + * @param metricName Metric name (will have _total suffix added automatically) + * @param value Counter value + * @param labels Label key-value pairs (must be even number of strings) + * @return This builder for chaining + */ + public MetricsDataBuilder counter(String metricName, long value, String... labels) { + ensureCurrentSnapshot(); + currentSnapshot.addCounter(metricName, value, parseLabels(labels)); + return this; + } + + /** + * Add a timer metric to the current snapshot. + * + * @param metricName Metric name + * @param latencies Array of latency values in milliseconds + * @param labels Label key-value pairs (must be even number of strings) + * @return This builder for chaining + */ + public MetricsDataBuilder timer(String metricName, long[] latencies, String... labels) { + ensureCurrentSnapshot(); + currentSnapshot.addTimer(metricName, latencies, parseLabels(labels)); + return this; + } + + /** + * Build and write the database file. + * + * @throws Exception If database creation fails + */ + public void build() throws Exception { + if (currentSnapshot != null) { + snapshots.add(currentSnapshot); + } + + if (snapshots.isEmpty()) { + throw new IllegalStateException("No snapshots defined. Use atTime() to add snapshots."); + } + + NBComponent root = new NBBaseComponent(null, NBLabels.forKV("builder", "test")); + String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath(); + + try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter( + root, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) { + + for (Snapshot snapshot : snapshots) { + snapshot.writeToReporter(reporter); + } + } + } + + private void ensureCurrentSnapshot() { + if (currentSnapshot == null) { + throw new IllegalStateException("No current snapshot. Call atTime() first."); + } + } + + private Map parseLabels(String... labels) { + if (labels.length % 2 != 0) { + throw new IllegalArgumentException("Labels must be key-value pairs (even number of strings)"); + } + + Map labelMap = new LinkedHashMap<>(); + for (int i = 0; i < labels.length; i += 2) { + labelMap.put(labels[i], labels[i + 1]); + } + return labelMap; + } + + /** + * Represents a single metrics snapshot at a specific point in time. + */ + private static class Snapshot { + private final Instant timestamp; + private final List counters = new ArrayList<>(); + private final List timers = new ArrayList<>(); + + Snapshot(Instant timestamp) { + this.timestamp = timestamp; + } + + void addCounter(String metricName, long value, Map labels) { + Map fullLabels = new LinkedHashMap<>(labels); + fullLabels.put("name", metricName); + + NBMetricCounter counter = new NBMetricCounter( + NBLabels.forMap(fullLabels), + "counter", + MetricCategory.Core + ); + counter.inc(value); + counters.add(counter); + } + + void addTimer(String metricName, long[] latenciesMs, Map labels) { + Map fullLabels = new LinkedHashMap<>(labels); + fullLabels.put("name", metricName); + + NBLabels nbLabels = NBLabels.forMap(fullLabels); + NBMetricTimer timer = new NBMetricTimer( + nbLabels, + new DeltaHdrHistogramReservoir(nbLabels, 3), + "timer", + MetricCategory.Core + ); + + for (long latencyMs : latenciesMs) { + timer.update(Duration.ofMillis(latencyMs)); + } + timers.add(timer); + } + + void writeToReporter(SqliteSnapshotReporter reporter) { + List allMetrics = new ArrayList<>(); + allMetrics.addAll(counters); + allMetrics.addAll(timers); + + @SuppressWarnings("unchecked") + List metrics = + (List) (List) allMetrics; + + MetricsView view = MetricsView.capture(metrics, 30000L); + + // Override timestamp by capturing at specific time + // Note: MetricsView uses current time, but reporter will use it + reporter.onMetricsSnapshot(view); + } + } + + /** + * Format an Instant as ISO-8601 for documentation. + */ + public static String formatTimestamp(Instant instant) { + return instant.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/TestDatabaseGenerator.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/TestDatabaseGenerator.java new file mode 100644 index 000000000..282094a40 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/TestDatabaseGenerator.java @@ -0,0 +1,329 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.testdata; + +import io.nosqlbench.nb.api.components.core.NBComponent; +import io.nosqlbench.nb.api.components.core.NBBaseComponent; +import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory; +import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter; +import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge; +import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer; +import io.nosqlbench.nb.api.engine.metrics.reporters.SqliteSnapshotReporter; +import io.nosqlbench.nb.api.engine.metrics.reporters.MetricInstanceFilter; +import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir; +import io.nosqlbench.nb.api.engine.metrics.view.MetricsView; +import io.nosqlbench.nb.api.labels.NBLabels; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Generates test databases with various metric scenarios for testing query commands. + * Each generator method creates a database with specific characteristics documented + * in TEST_DATABASES.md. + */ +public class TestDatabaseGenerator { + + /** + * Generate simple_counter.db + * + * A basic counter metric with two label dimensions over 5 minutes. + * + * Schema: + * - Metric: activity_ops_total (COUNTER) + * - Labels: activity={read,write}, host=server1 + * - Time range: 5 minutes (10 snapshots @ 30s intervals) + * - Values: Linear growth (read: 0→100, write: 0→150) + * + * Purpose: Test instant queries, simple ranges, basic rate calculations + */ + public static void generateSimpleCounter(Path outputPath, NBComponent parent) throws Exception { + String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath(); + + try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter( + parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) { + + // Base timestamp: 2025-10-23 10:00:00 UTC + long baseTime = 1729681200000L; + long intervalMs = 30000L; + + for (int i = 0; i < 10; i++) { + List counters = new ArrayList<>(); + + // Create read counter + NBMetricCounter readCounter = new NBMetricCounter( + NBLabels.forKV("name", "activity_ops", "activity", "read", "host", "server1"), + "ops", + MetricCategory.Core + ); + readCounter.inc(i * 10); // 0, 10, 20, ..., 90 + + // Create write counter + NBMetricCounter writeCounter = new NBMetricCounter( + NBLabels.forKV("name", "activity_ops", "activity", "write", "host", "server1"), + "ops", + MetricCategory.Core + ); + writeCounter.inc(i * 15); // 0, 15, 30, ..., 135 + + counters.add(readCounter); + counters.add(writeCounter); + + // Create MetricsView and report snapshot + // Note: Timestamps will be auto-generated by the reporter + MetricsView view = MetricsView.capture(counters, intervalMs); + reporter.onMetricsSnapshot(view); + + // Sleep to create distinct timestamps + Thread.sleep(100); + } + } + } + + /** + * Generate multi_dimensional.db + * + * Complex metrics with 4 dimensional labels for testing grouping and filtering. + * + * Schema: + * - Metric: request_total (COUNTER) + * - Labels: activity={read,write}, host={server1,server2,server3}, + * region={us-east,us-west}, env={prod,staging} + * - Combinations: 2 * 3 * 2 * 2 = 24 unique label sets + * - Time range: 2 minutes (4 snapshots @ 30s) + * + * Purpose: Test label filtering, aggregations with grouping, topk queries + */ + public static void generateMultiDimensional(Path outputPath, NBComponent parent) throws Exception { + String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath(); + + try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter( + parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) { + + long baseTime = 1729681200000L; + long intervalMs = 30000L; + String[] activities = {"read", "write"}; + String[] hosts = {"server1", "server2", "server3"}; + String[] regions = {"us-east", "us-west"}; + String[] envs = {"prod", "staging"}; + + for (int snapshot = 0; snapshot < 4; snapshot++) { + List counters = new ArrayList<>(); + int counter = 0; + + for (String activity : activities) { + for (String host : hosts) { + for (String region : regions) { + for (String env : envs) { + NBMetricCounter metric = new NBMetricCounter( + NBLabels.forKV( + "name", "requests", + "activity", activity, + "host", host, + "region", region, + "env", env + ), + "requests", + MetricCategory.Core + ); + + // Value varies by combination and snapshot + long value = (counter * 10) + (snapshot * 100); + metric.inc(value); + counters.add(metric); + counter++; + } + } + } + } + + MetricsView view = MetricsView.capture(counters, intervalMs); + reporter.onMetricsSnapshot(view); + Thread.sleep(100); + } + } + } + + /** + * Generate latency_timers.db + * + * Summary/timer metrics with quantiles for latency analysis. + * + * Schema: + * - Metric: operation_latency_ms (TIMER/SUMMARY) + * - Labels: operation={select,insert,update,delete} + * - Quantiles: p50, p75, p90, p95, p98, p99, p999 + * - Statistics: count, sum, min, max, mean, stddev + * - Time range: 3 minutes (6 snapshots @ 30s) + * + * Purpose: Test quantile queries, statistical aggregations + */ + public static void generateLatencyTimers(Path outputPath, NBComponent parent) throws Exception { + String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath(); + + try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter( + parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) { + + long baseTime = 1729681200000L; + long intervalMs = 30000L; + String[] operations = {"select", "insert", "update", "delete"}; + + for (int snapshot = 0; snapshot < 6; snapshot++) { + List timers = new ArrayList<>(); + + for (int opIdx = 0; opIdx < operations.length; opIdx++) { + String operation = operations[opIdx]; + NBLabels labels = NBLabels.forKV( + "name", "operation_latency", + "operation", operation + ); + + NBMetricTimer timer = new NBMetricTimer( + labels, + new DeltaHdrHistogramReservoir(labels, 3), + "latency", + MetricCategory.Core + ); + + // Vary latencies by operation type and over time + long baseLatency = (opIdx + 1) * 10_000_000L; // 10ms, 20ms, 30ms, 40ms in nanoseconds + long drift = snapshot * 2_000_000L; // 2ms increase per snapshot + + // Record latencies with some variation + int recordCount = 100 + (snapshot * 20); + for (int i = 0; i < recordCount; i++) { + // Create a distribution of latencies + long latency; + if (i < recordCount * 0.5) { + latency = baseLatency + drift; // p50 + } else if (i < recordCount * 0.75) { + latency = (long) ((baseLatency + drift) * 1.5); // p75 + } else if (i < recordCount * 0.95) { + latency = (long) ((baseLatency + drift) * 2.0); // p95 + } else { + latency = (long) ((baseLatency + drift) * 3.0); // p99 + } + timer.update(Duration.ofNanos(latency)); + } + + timers.add(timer); + } + + long timestamp = baseTime + (snapshot * intervalMs); + MetricsView view = MetricsView.capture(timers, intervalMs); + reporter.onMetricsSnapshot(view); + Thread.sleep(100); + } + } + } + + /** + * Generate rate_calculations.db + * + * Counter metrics with predictable growth patterns for testing rate() and increase(). + * + * Schema: + * - Metrics: counter_total (COUNTER) + * - Labels: pattern={linear,exponential,step} + * - Time range: 1 hour (120 snapshots @ 30s) + * - Patterns: + * - linear: grows by 10 per snapshot (constant rate) + * - exponential: grows by 1.01x per snapshot + * - step: 100 for 10 snapshots, then jump to 200 for 10, etc. + * + * Purpose: Test rate() and increase() with known expected results + */ + public static void generateRateCalculations(Path outputPath, NBComponent parent) throws Exception { + String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath(); + + try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter( + parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) { + + long baseTime = 1729681200000L; + long intervalMs = 30000L; + double exponentialValue = 100.0; + + for (int i = 0; i < 120; i++) { + List counters = new ArrayList<>(); + + // Linear counter: constant rate of 10 per 30s = 0.333 ops/sec + NBMetricCounter linearCounter = new NBMetricCounter( + NBLabels.forKV("name", "patterns", "pattern", "linear"), + "counter", + MetricCategory.Core + ); + linearCounter.inc(i * 10); + counters.add(linearCounter); + + // Exponential counter: exponential growth + exponentialValue *= 1.01; // 1% growth per snapshot + NBMetricCounter expCounter = new NBMetricCounter( + NBLabels.forKV("name", "patterns", "pattern", "exponential"), + "counter", + MetricCategory.Core + ); + expCounter.inc((long) exponentialValue); + counters.add(expCounter); + + // Step counter: steps every 10 snapshots + long stepValue = ((i / 10) + 1) * 100; + NBMetricCounter stepCounter = new NBMetricCounter( + NBLabels.forKV("name", "patterns", "pattern", "step"), + "counter", + MetricCategory.Core + ); + stepCounter.inc(stepValue); + counters.add(stepCounter); + + MetricsView view = MetricsView.capture(counters, intervalMs); + reporter.onMetricsSnapshot(view); + Thread.sleep(50); // Shorter sleep for many snapshots + } + } + } + + /** + * Main method to generate all test databases. + * Run this to regenerate test data after schema changes. + */ + public static void main(String[] args) throws Exception { + Path testDataDir = Path.of("nb-apis/nb-mql-api/src/test/resources/testdata"); + + // Create a minimal NBComponent for the reporter + NBComponent root = new NBBaseComponent(null, NBLabels.forKV("generator", "testdata")); + + System.out.println("Generating test databases..."); + + generateSimpleCounter(testDataDir.resolve("simple_counter.db"), root); + System.out.println("✓ Generated simple_counter.db"); + + generateMultiDimensional(testDataDir.resolve("multi_dimensional.db"), root); + System.out.println("✓ Generated multi_dimensional.db"); + + generateLatencyTimers(testDataDir.resolve("latency_timers.db"), root); + System.out.println("✓ Generated latency_timers.db"); + + generateRateCalculations(testDataDir.resolve("rate_calculations.db"), root); + System.out.println("✓ Generated rate_calculations.db"); + + System.out.println("\nAll test databases generated successfully!"); + System.out.println("Location: " + testDataDir.toAbsolutePath()); + } +} diff --git a/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/TestDatabaseLoader.java b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/TestDatabaseLoader.java new file mode 100644 index 000000000..919ea6a77 --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/java/io/nosqlbench/nb/mql/testdata/TestDatabaseLoader.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.nb.mql.testdata; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Utility for loading test databases from src/test/resources/testdata/. + * Provides easy access to pre-generated test databases for unit and integration tests. + */ +public class TestDatabaseLoader { + + private static final String TEST_DATA_DIR = "testdata"; + + /** + * Available test databases with their purposes. + */ + public enum TestDatabase { + SIMPLE_COUNTER("simple_counter.db", "Basic counters for instant/rate queries"), + MULTI_DIMENSIONAL("multi_dimensional.db", "Complex 4D labels for aggregations"), + LATENCY_TIMERS("latency_timers.db", "Timer metrics with quantiles"), + RATE_CALCULATIONS("rate_calculations.db", "Predictable growth patterns for rate testing"); + + private final String filename; + private final String description; + + TestDatabase(String filename, String description) { + this.filename = filename; + this.description = description; + } + + public String getFilename() { + return filename; + } + + public String getDescription() { + return description; + } + } + + /** + * Get the path to a test database by enum. + * + * @param database The test database enum + * @return Path to the database file + * @throws IOException If the database file cannot be found + */ + public static Path getDatabase(TestDatabase database) throws IOException { + return getDatabase(database.getFilename()); + } + + /** + * Get the path to a test database by filename. + * + * @param filename Name of the database file (e.g., "simple_counter.db") + * @return Path to the database file + * @throws IOException If the database file cannot be found + */ + public static Path getDatabase(String filename) throws IOException { + String resourcePath = TEST_DATA_DIR + "/" + filename; + URL resource = TestDatabaseLoader.class.getClassLoader().getResource(resourcePath); + + if (resource == null) { + throw new IOException( + "Test database not found: " + resourcePath + "\n" + + "Make sure test databases have been generated by running TestDatabaseGenerator.main()" + ); + } + + try { + return Paths.get(resource.toURI()); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI for test database: " + resourcePath, e); + } + } + + /** + * Check if a test database exists. + * + * @param database The test database enum + * @return true if the database file exists + */ + public static boolean exists(TestDatabase database) { + try { + getDatabase(database); + return true; + } catch (IOException e) { + return false; + } + } + + /** + * Get a descriptive message about available test databases. + * + * @return Multi-line string describing all test databases + */ + public static String getAvailableDatabases() { + StringBuilder sb = new StringBuilder("Available test databases:\n"); + for (TestDatabase db : TestDatabase.values()) { + sb.append(String.format(" - %s: %s\n", + db.getFilename(), + db.getDescription())); + } + return sb.toString(); + } + + private TestDatabaseLoader() { + // Utility class - prevent instantiation + } +} diff --git a/nb-apis/nb-mql-api/src/test/resources/testdata/TEST_DATABASES.md b/nb-apis/nb-mql-api/src/test/resources/testdata/TEST_DATABASES.md new file mode 100644 index 000000000..cf9cc878d --- /dev/null +++ b/nb-apis/nb-mql-api/src/test/resources/testdata/TEST_DATABASES.md @@ -0,0 +1,310 @@ +# Test Database Documentation + +This directory contains pre-generated SQLite metrics databases for testing the MetricsQL query engine. Each database represents a specific scenario with known data patterns. + +## Overview + +All databases use the NoSQLBench OpenMetrics-aligned schema created by `SqliteSnapshotReporter`. They contain realistic metric data generated using actual metric instruments (`NBMetricCounter`, `NBMetricTimer`). + +--- + +## simple_counter.db + +### Purpose +Test basic counter queries, instant values, simple rate calculations, and time-series ranges. + +### Schema +- **Metric Family**: `activity_ops` +- **Sample**: `activity_ops_total` (COUNTER) +- **Type**: COUNTER + +### Label Dimensions +- `activity`: {read, write} +- `host`: {server1} + +### Data Characteristics +- **Snapshots**: 10 (taken ~100ms apart during generation) +- **Label Combinations**: 2 (read, write) +- **Total Samples**: 20 +- **Growth Pattern**: Linear increments + - `activity=read`: increments by 10 per snapshot (0, 10, 20, ..., 90) + - `activity=write`: increments by 15 per snapshot (0, 15, 30, ..., 135) + +### Expected Query Results + +**Instant Query (latest value):** +```sql +SELECT sv.value, lv.value as activity +FROM sample_value sv +JOIN sample_name sn ON sv.sample_name_id = sn.id +JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id +JOIN label_key lk ON lsm.label_key_id = lk.id +JOIN label_value lv ON lsm.label_value_id = lv.id +WHERE sn.sample = 'activity_ops_total' + AND lk.name = 'activity' + AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) +ORDER BY activity; +``` +Expected: `{activity=read: 90, activity=write: 135}` + +**Count Total Samples:** +```sql +SELECT COUNT(*) FROM sample_value; +``` +Expected: `20` + +**Rate Calculation (approximate):** +For `activity=read`, the rate should be approximately **10 ops per 100ms** = **100 ops/sec** (simplified; actual timing varies slightly) + +### Use Cases +- Testing instant value queries +- Basic label filtering +- Simple rate calculations +- Time-series data retrieval + +--- + +## multi_dimensional.db + +### Purpose +Test complex label filtering, multi-dimensional aggregations, grouping operations, and topk queries. + +### Schema +- **Metric Family**: `requests` +- **Sample**: `request_total` (COUNTER) +- **Type**: COUNTER + +### Label Dimensions +- `activity`: {read, write} +- `host`: {server1, server2, server3} +- `region`: {us-east, us-west} +- `env`: {prod, staging} + +### Data Characteristics +- **Snapshots**: 4 +- **Label Combinations**: 24 (2 × 3 × 2 × 2) +- **Total Samples**: 96 +- **Value Pattern**: `(combo_index * 10) + (snapshot_index * 100)` + - First snapshot: values 0, 10, 20, ..., 230 + - Second snapshot: values 100, 110, 120, ..., 330 + - And so on... + +### Expected Query Results + +**Count by Environment:** +```sql +SELECT lv.value as env, COUNT(*) as sample_count +FROM sample_value sv +JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id +JOIN label_key lk ON lsm.label_key_id = lk.id +JOIN label_value lv ON lsm.label_value_id = lv.id +WHERE lk.name = 'env' +GROUP BY env; +``` +Expected: `{prod: 48, staging: 48}` + +**Unique Label Sets:** +```sql +SELECT COUNT(DISTINCT label_set_id) FROM sample_value; +``` +Expected: `24` + +### Use Cases +- Multi-dimensional filtering (e.g., `activity=write AND region=us-east AND env=prod`) +- Aggregations with grouping (e.g., `sum by (region, env)`) +- Top-K queries (e.g., "top 5 hosts by request count") +- Complex label pattern matching + +--- + +## latency_timers.db + +### Purpose +Test summary/timer metrics with quantiles, statistical aggregations, and rate statistics. + +### Schema +- **Metric Family**: `operation_latency` +- **Sample**: `operation_latency_ms` (TIMER/SUMMARY) +- **Type**: TIMER + +### Label Dimensions +- `operation`: {select, insert, update, delete} + +### Data Characteristics +- **Snapshots**: 6 +- **Operations**: 4 +- **Total Sample Values**: 24 +- **Quantiles**: p50, p75, p90, p95, p98, p99, p999 (stored in `sample_quantile` table) +- **Statistics**: count, sum, min, max, mean, stddev (in `sample_value`) +- **Rate Statistics**: mean, m1, m5, m15 (in `sample_rate` table) + +### Latency Patterns +Base latencies by operation (in nanoseconds): +- `select`: 10ms base, increases by 2ms per snapshot +- `insert`: 20ms base, increases by 2ms per snapshot +- `update`: 30ms base, increases by 2ms per snapshot +- `delete`: 40ms base, increases by 2ms per snapshot + +Distribution within each snapshot: +- 50% of samples at base latency (p50) +- 25% at 1.5× base (p75) +- 20% at 2.0× base (p95) +- 5% at 3.0× base (p99) + +### Expected Query Results + +**Get p95 latency for select operations:** +```sql +SELECT sq.quantile_value +FROM sample_quantile sq +JOIN sample_value sv ON sq.sample_value_id = sv.id +JOIN sample_name sn ON sv.sample_name_id = sn.id +JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id +JOIN label_key lk ON lsm.label_key_id = lk.id +JOIN label_value lv ON lsm.label_value_id = lv.id +WHERE sn.sample = 'operation_latency_ms' + AND lk.name = 'operation' + AND lv.value = 'select' + AND sq.quantile = 0.95 + AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value) +LIMIT 1; +``` +Expected: Approximately `22,000,000` nanoseconds (22ms) for latest snapshot + +**Count quantile entries:** +```sql +SELECT COUNT(*) FROM sample_quantile; +``` +Expected: `168` (24 samples × 7 quantiles each) + +### Use Cases +- Quantile extraction (p50, p95, p99) +- Statistical aggregations (mean, stddev, min, max) +- Latency trend analysis over time +- Rate statistics queries (m1, m5, m15) + +--- + +## rate_calculations.db + +### Purpose +Test rate() and increase() commands with predictable, mathematically known patterns over extended time periods. + +### Schema +- **Metric Family**: `patterns` +- **Sample**: `counter_total` (COUNTER) +- **Type**: COUNTER + +### Label Dimensions +- `pattern`: {linear, exponential, step} + +### Data Characteristics +- **Snapshots**: 120 (simulating 1 hour at 30s intervals) +- **Patterns**: 3 +- **Total Samples**: 360 +- **Interval Between Snapshots**: ~50ms (during generation) + +### Growth Patterns + +#### Linear Pattern (`pattern=linear`) +- Growth: **+10 per snapshot** +- Values: 0, 10, 20, 30, ..., 1190 +- Expected rate: **10 ops / 50ms** = **200 ops/sec** (constant) +- Total increase over all snapshots: **1190** + +#### Exponential Pattern (`pattern=exponential`) +- Growth: **×1.01 per snapshot** (1% compound growth) +- Starting value: 100 +- Values: 100, 101, 102.01, 103.03, ... +- Expected rate: Increases over time +- Formula: `value[i] = 100 × (1.01)^i` + +#### Step Pattern (`pattern=step`) +- Growth: **Steps every 10 snapshots** +- Values: 100 (snapshots 0-9), 200 (10-19), 300 (20-29), ..., 1200 (110-119) +- Expected rate: **0 within step, jump at boundaries** +- Total steps: 12 + +### Expected Query Results + +**Linear pattern increase over all snapshots:** +```sql +SELECT MAX(sv.value) - MIN(sv.value) as total_increase +FROM sample_value sv +JOIN sample_name sn ON sv.sample_name_id = sn.id +JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id +JOIN label_key lk ON lsm.label_key_id = lk.id +JOIN label_value lv ON lsm.label_value_id = lv.id +WHERE sn.sample = 'counter_total' + AND lk.name = 'pattern' + AND lv.value = 'linear'; +``` +Expected: `1190` + +**Count snapshots per pattern:** +```sql +SELECT lv.value as pattern, COUNT(*) as snapshots +FROM sample_value sv +JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id +JOIN label_key lk ON lsm.label_key_id = lk.id +JOIN label_value lv ON lsm.label_value_id = lv.id +WHERE lk.name = 'pattern' +GROUP BY pattern; +``` +Expected: `{linear: 120, exponential: 120, step: 120}` + +### Use Cases +- Testing rate() with constant rate (linear) +- Testing rate() with changing rate (exponential) +- Testing rate() with discontinuous data (step) +- Testing increase() over various time windows +- Validating rate calculation algorithms with known results + +--- + +## Schema Reference + +All databases share this schema (created by `SqliteSnapshotReporter`): + +``` +metric_family (id, name, help, unit, type) + └─ sample_name (id, metric_family_id, sample) + └─ sample_value (id, sample_name_id, label_set_id, timestamp_ms, value, count, sum, min, max, mean, stddev) + ├─ sample_quantile (sample_value_id, quantile, quantile_value) + ├─ sample_rate (sample_value_id, rate_type, rate_value) + └─ sample_histogram (sample_value_id, start_seconds, interval_seconds, max_value, histogram_base64) + +label_key (id, name) +label_value (id, value) +label_set (id, hash) +label_set_membership (label_set_id, label_key_id, label_value_id) +``` + +## Regenerating Test Databases + +To regenerate all test databases (e.g., after schema changes): + +```bash +mvn exec:java -pl nb-apis/nb-mql-api \ + -Dexec.mainClass="io.nosqlbench.nb.mql.testdata.TestDatabaseGenerator" \ + -Dexec.classpathScope=test +``` + +The generator will overwrite existing databases in `src/test/resources/testdata/`. + +## Using in Tests + +Use `TestDatabaseLoader` utility to access these databases in your tests: + +```java +import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader; + +Path dbPath = TestDatabaseLoader.getDatabase("simple_counter.db"); +Connection conn = MetricsDatabaseReader.connect(dbPath); +// ... run queries +``` + +--- + +*Generated: 2025-10-23* +*Schema Version: OpenMetrics-aligned (SqliteSnapshotReporter)* diff --git a/nb-apis/nb-mql-api/src/test/resources/testdata/examples.db b/nb-apis/nb-mql-api/src/test/resources/testdata/examples.db new file mode 100644 index 000000000..c0ee94fb2 Binary files /dev/null and b/nb-apis/nb-mql-api/src/test/resources/testdata/examples.db differ diff --git a/nb-apis/nb-mql-api/src/test/resources/testdata/latency_timers.db b/nb-apis/nb-mql-api/src/test/resources/testdata/latency_timers.db new file mode 100644 index 000000000..affe4eaec Binary files /dev/null and b/nb-apis/nb-mql-api/src/test/resources/testdata/latency_timers.db differ diff --git a/nb-apis/nb-mql-api/src/test/resources/testdata/multi_dimensional.db b/nb-apis/nb-mql-api/src/test/resources/testdata/multi_dimensional.db new file mode 100644 index 000000000..4bbcfe9af Binary files /dev/null and b/nb-apis/nb-mql-api/src/test/resources/testdata/multi_dimensional.db differ diff --git a/nb-apis/nb-mql-api/src/test/resources/testdata/rate_calculations.db b/nb-apis/nb-mql-api/src/test/resources/testdata/rate_calculations.db new file mode 100644 index 000000000..03708821c Binary files /dev/null and b/nb-apis/nb-mql-api/src/test/resources/testdata/rate_calculations.db differ diff --git a/nb-apis/nb-mql-api/src/test/resources/testdata/simple_counter.db b/nb-apis/nb-mql-api/src/test/resources/testdata/simple_counter.db new file mode 100644 index 000000000..1775836ca Binary files /dev/null and b/nb-apis/nb-mql-api/src/test/resources/testdata/simple_counter.db differ diff --git a/nb-apis/pom.xml b/nb-apis/pom.xml index 17f830ccb..54f787474 100644 --- a/nb-apis/pom.xml +++ b/nb-apis/pom.xml @@ -35,6 +35,7 @@ adapters-api expr-api nb-api + nb-mql-api nb-apis-all diff --git a/nb-engine/nb-engine-all/pom.xml b/nb-engine/nb-engine-all/pom.xml index 1d175b2b0..e69b108a3 100644 --- a/nb-engine/nb-engine-all/pom.xml +++ b/nb-engine/nb-engine-all/pom.xml @@ -55,6 +55,12 @@ nb-engine-extensions ${revision} + + + io.nosqlbench + nb-mql-api + ${revision} + diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 32bbcfe04..4a324c772 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -182,6 +182,11 @@ public class NBCLI implements Function, NBLabeledElement { .activate(); ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig); // THIS should be the first time log4j2 is invoked! + // Add shutdown handler to timestamp symlinks when session ends + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + NBCLI.loggerConfig.timestampSymlinksOnShutdown(); + }, "symlink-timestamp-shutdown")); + NBCLI.logger = LogManager.getLogger("NBCLI"); // TODO: Detect if the logger config was already initialized (error) NBCLI.loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO")); if (NBCLI.logger.isInfoEnabled()) diff --git a/nb-engine/nb-engine-cli/src/main/resources/metrics_reporting.md b/nb-engine/nb-engine-cli/src/main/resources/metrics_reporting.md index ac8fa71be..79b2f1f69 100644 --- a/nb-engine/nb-engine-cli/src/main/resources/metrics_reporting.md +++ b/nb-engine/nb-engine-cli/src/main/resources/metrics_reporting.md @@ -1,4 +1,4 @@ -# Metrics Reporting Guide +contin# Metrics Reporting Guide ## What You Can Configure diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/logging/NBLoggerConfig.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/logging/NBLoggerConfig.java index 4fe085e79..1a947caaa 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/logging/NBLoggerConfig.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/core/logging/NBLoggerConfig.java @@ -485,4 +485,42 @@ public class NBLoggerConfig extends ConfigurationFactory { } + + /** + * Update modification time on symlinks to reflect shutdown time. + * Called by shutdown hook to touch symlinks so their mtime shows when the session ended. + * Updates symlinks for: scenario.log, status.log, runtime.log, verify.log, metrics.db, heartbeat.log + */ + public void timestampSymlinksOnShutdown() { + if (loggerDir == null) { + return; + } + + // List of symlinks to update + String[] symlinks = { + "scenario.log", + "status.log", + "runtime.log", + "verify.log", + "metrics.db", + "heartbeat.log" + }; + + for (String linkName : symlinks) { + try { + Path linkPath = loggerDir.resolve(linkName); + + if (!Files.exists(linkPath, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + continue; // Symlink doesn't exist, skip + } + + // Update the modification time of the symlink itself (not the target file) + Files.setLastModifiedTime(linkPath, + java.nio.file.attribute.FileTime.from(java.time.Instant.now())); + + } catch (IOException | UnsupportedOperationException e) { + // Silently ignore - symlinks may not be supported or may have issues during shutdown + } + } + } }