mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2026-10-12 09:24:52 -05:00
- New nb-mql-api module with 11 query commands (instant, range, rate, increase, quantile, agg, ratio, topk, session, summary, sql)
- BundledApp integration (@Service selector="mql") accessible via nb5 CLI - 6 output formats (table, csv, tsv, json, jsonl, markdown) with enum-based architecture - 107 passing tests with self-validating documentation and 5 test databases - Add sqlite-jdbc to dependency management and nb-mql-api to nb-engine-all - Add shutdown hook to update symlink mtimes on session end
This commit is contained in:
1 parent
68c35027c5
commit
30c8a9d015
79 files changed
+9825
-2
No files matched your search
@@ -181,6 +181,11 @@
|
||||
<artifactId>ascii-data</artifactId>
|
||||
<version>1.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.xerial</groupId>
|
||||
<artifactId>sqlite-jdbc</artifactId>
|
||||
<version>3.46.0.0</version>
|
||||
</dependency>
|
||||
<!-- For CQL compression option -->
|
||||
<dependency>
|
||||
<groupId>org.lz4</groupId>
|
||||
|
||||
@@ -159,7 +159,6 @@
|
||||
<dependency>
|
||||
<groupId>org.xerial</groupId>
|
||||
<artifactId>sqlite-jdbc</artifactId>
|
||||
<version>3.46.0.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
@@ -47,5 +47,10 @@
|
||||
<artifactId>adapters-api</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.nosqlbench</groupId>
|
||||
<artifactId>nb-mql-api</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -0,0 +1,139 @@
|
||||
<!--
|
||||
~ Copyright (c) nosqlbench
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>mvn-defaults</artifactId>
|
||||
<groupId>io.nosqlbench</groupId>
|
||||
<version>${revision}</version>
|
||||
<relativePath>../../mvn-defaults</relativePath>
|
||||
</parent>
|
||||
|
||||
<artifactId>nb-mql-api</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>
|
||||
MetricsQL-inspired query engine for NoSQLBench SQLite metrics databases.
|
||||
Provides read-only query commands for analyzing metrics data with support
|
||||
for rates, aggregations, quantiles, and time-series analysis.
|
||||
</description>
|
||||
|
||||
<dependencies>
|
||||
<!-- NoSQLBench API for labels and base types -->
|
||||
<dependency>
|
||||
<groupId>io.nosqlbench</groupId>
|
||||
<artifactId>nb-api</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- SQLite JDBC driver (version managed in parent) -->
|
||||
<dependency>
|
||||
<groupId>org.xerial</groupId>
|
||||
<artifactId>sqlite-jdbc</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- CLI framework (picocli) - inherited from parent -->
|
||||
<dependency>
|
||||
<groupId>info.picocli</groupId>
|
||||
<artifactId>picocli</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- JSON output formatting -->
|
||||
<dependency>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- CSV output formatting -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-csv</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Logging for standalone JAR (sqlite-jdbc needs SLF4J) -->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>2.0.16</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-slf4j2-impl</artifactId>
|
||||
<version>2.25.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<version>2.25.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<version>2.25.2</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Testing dependencies inherited from parent -->
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<testResources>
|
||||
<testResource>
|
||||
<directory>src/test/resources</directory>
|
||||
<filtering>false</filtering>
|
||||
<includes>
|
||||
<include>**/*.db</include>
|
||||
</includes>
|
||||
</testResource>
|
||||
<testResource>
|
||||
<directory>src/test/resources</directory>
|
||||
<filtering>true</filtering>
|
||||
<excludes>
|
||||
<exclude>**/*.db</exclude>
|
||||
</excludes>
|
||||
</testResource>
|
||||
</testResources>
|
||||
|
||||
<plugins>
|
||||
<!-- Maven Assembly Plugin for standalone JAR -->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>io.nosqlbench.nb.mql.cli.MetricsQLCLI</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
<finalName>nb-metricsql</finalName>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql;
|
||||
|
||||
import io.nosqlbench.nb.annotations.Service;
|
||||
import io.nosqlbench.nb.api.apps.BundledApp;
|
||||
import io.nosqlbench.nb.mql.cli.MetricsQLCLI;
|
||||
import picocli.CommandLine;
|
||||
|
||||
/**
|
||||
* MetricsQL Query Engine as a NoSQLBench BundledApp.
|
||||
* Allows querying SQLite metrics databases using MetricsQL-inspired syntax.
|
||||
*
|
||||
* Usage:
|
||||
* nb5 mql instant --metric ops_total
|
||||
* nb5 mql range --metric ops_total --window 5m
|
||||
*
|
||||
* This integrates the MetricsQL CLI as a bundled application within NoSQLBench,
|
||||
* making it immediately available without needing a separate tool.
|
||||
*/
|
||||
@Service(value = BundledApp.class, selector = "mql")
|
||||
public class MQLApp implements BundledApp {
|
||||
|
||||
@Override
|
||||
public int applyAsInt(String[] args) {
|
||||
// Delegate to the picocli-based MetricsQLCLI
|
||||
// This allows all the command routing and validation to work seamlessly
|
||||
return new CommandLine(new MetricsQLCLI()).execute(args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution (optional).
|
||||
* Can also be run via: java -cp ... io.nosqlbench.nb.mql.MQLApp instant --metric ops
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
MQLApp app = new MQLApp();
|
||||
int exitCode = app.applyAsInt(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.OutputFormat;
|
||||
import io.nosqlbench.nb.mql.format.ResultFormatter;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/**
|
||||
* Abstract base class for all query command CLIs.
|
||||
* Provides common functionality for database connection, formatting, and error handling.
|
||||
*
|
||||
* Subclasses must implement:
|
||||
* - createCommand() - Create the specific MetricsQueryCommand instance
|
||||
* - buildParams() - Build the parameter map for the command
|
||||
*/
|
||||
public abstract class AbstractQueryCommandCLI implements Callable<Integer> {
|
||||
|
||||
@Option(
|
||||
names = {"-d", "--db"},
|
||||
description = "Path to metrics database (default: logs/metrics.db)",
|
||||
defaultValue = "logs/metrics.db"
|
||||
)
|
||||
protected Path databasePath;
|
||||
|
||||
@Option(
|
||||
names = {"-m", "--metric"},
|
||||
description = "Metric name to query",
|
||||
required = true
|
||||
)
|
||||
protected String metric;
|
||||
|
||||
@Option(
|
||||
names = {"-l", "--labels"},
|
||||
description = "Label filters as key=value,key=value",
|
||||
split = ","
|
||||
)
|
||||
protected Map<String, String> labels = new LinkedHashMap<>();
|
||||
|
||||
@Option(
|
||||
names = {"-f", "--format"},
|
||||
description = "Output format: table, json, jsonl, csv, tsv, markdown",
|
||||
defaultValue = "table"
|
||||
)
|
||||
protected OutputFormat format = OutputFormat.TABLE;
|
||||
|
||||
@Option(
|
||||
names = {"--explain"},
|
||||
description = "Show the SQL query instead of executing"
|
||||
)
|
||||
protected boolean explain;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
|
||||
MetricsQueryCommand command = createCommand();
|
||||
Map<String, Object> params = buildParams();
|
||||
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
if (explain) {
|
||||
System.out.println("SQL Query:");
|
||||
System.out.println(result.executedSQL());
|
||||
return 0;
|
||||
}
|
||||
|
||||
ResultFormatter formatter = format.createFormatter();
|
||||
System.out.println(formatter.format(result));
|
||||
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error: " + e.getMessage());
|
||||
if (System.getenv("DEBUG") != null) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the specific command instance for this CLI.
|
||||
*
|
||||
* @return MetricsQueryCommand instance
|
||||
*/
|
||||
protected abstract MetricsQueryCommand createCommand();
|
||||
|
||||
/**
|
||||
* Build the parameter map for the command.
|
||||
* Subclasses override to add command-specific parameters.
|
||||
*
|
||||
* @return Parameter map
|
||||
*/
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = new LinkedHashMap<>();
|
||||
params.put("metric", metric);
|
||||
if (!labels.isEmpty()) {
|
||||
params.put("labels", labels);
|
||||
}
|
||||
return params;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.AggregateCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for aggregate query command.
|
||||
* Performs aggregations (sum, avg, max, min, count) with optional grouping.
|
||||
*/
|
||||
@Command(
|
||||
name = "agg",
|
||||
description = "Perform aggregations (sum, avg, max, min, count) on metrics",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class AggregateCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Option(
|
||||
names = {"--func", "--function"},
|
||||
description = "Aggregate function: sum, avg, max, min, count",
|
||||
required = true
|
||||
)
|
||||
private String function;
|
||||
|
||||
@Option(
|
||||
names = {"--by"},
|
||||
description = "Group by labels (comma-separated, e.g., activity,host)",
|
||||
split = ","
|
||||
)
|
||||
private List<String> groupBy;
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h) - optional, defaults to latest snapshot"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new AggregateCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = super.buildParams();
|
||||
|
||||
params.put("function", function);
|
||||
|
||||
if (groupBy != null && !groupBy.isEmpty()) {
|
||||
params.put("by", groupBy);
|
||||
}
|
||||
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new AggregateCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.IncreaseCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for increase query command.
|
||||
* Calculates total increase in counter value over a time window.
|
||||
*/
|
||||
@Command(
|
||||
name = "increase",
|
||||
description = "Calculate total increase in counter over time window",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class IncreaseCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h, 30s)"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window (e.g., last 5m)"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new IncreaseCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = super.buildParams();
|
||||
|
||||
// Add time window parameter
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new IncreaseCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.InstantCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for instant query command.
|
||||
* Can be run standalone: java -cp ... io.nosqlbench.nb.mql.cli.InstantCLI --metric ops_total
|
||||
*/
|
||||
@Command(
|
||||
name = "instant",
|
||||
description = "Get the most recent value for a metric",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class InstantCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new InstantCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
// Use base class params (metric and labels)
|
||||
return super.buildParams();
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new InstantCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
|
||||
/**
|
||||
* Main entry point for the MetricsQL CLI tool.
|
||||
* Routes to subcommands for different query types.
|
||||
*
|
||||
* Usage:
|
||||
* nb-metricsql instant --metric ops_total
|
||||
* nb-metricsql range --metric ops_total --window 5m
|
||||
* nb-metricsql rate --metric ops_total --window 5m
|
||||
* ...
|
||||
*/
|
||||
@Command(
|
||||
name = "nb-metricsql",
|
||||
description = "MetricsQL query engine for NoSQLBench SQLite metrics databases",
|
||||
mixinStandardHelpOptions = true,
|
||||
version = "1.0.0",
|
||||
subcommands = {
|
||||
SessionCLI.class,
|
||||
SummaryCLI.class,
|
||||
InstantCLI.class,
|
||||
RangeCLI.class,
|
||||
RateCLI.class,
|
||||
IncreaseCLI.class,
|
||||
QuantileCLI.class,
|
||||
AggregateCLI.class,
|
||||
RatioCLI.class,
|
||||
TopKCLI.class,
|
||||
SqlCLI.class
|
||||
// All commands implemented except:
|
||||
// BinaryOpCLI.class (optional - advanced arithmetic)
|
||||
}
|
||||
)
|
||||
public class MetricsQLCLI implements Runnable {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// If no subcommand specified, show help
|
||||
CommandLine.usage(this, System.out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point for the standalone JAR.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new MetricsQLCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.QuantileCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for quantile query command.
|
||||
* Extracts percentile values from timer/summary metrics.
|
||||
*/
|
||||
@Command(
|
||||
name = "quantile",
|
||||
description = "Extract percentile values from timer/summary metrics",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class QuantileCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Option(
|
||||
names = {"-p", "--percentile", "--quantile"},
|
||||
description = "Quantile value (0.0-1.0, e.g., 0.95 for p95)",
|
||||
required = true
|
||||
)
|
||||
private double quantile;
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h) - optional, defaults to all data"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window (e.g., last 5m)"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new QuantileCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = super.buildParams();
|
||||
|
||||
params.put("quantile", quantile);
|
||||
|
||||
// Add time window if specified
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new QuantileCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.RangeCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for range query command.
|
||||
* Can be run standalone: java -cp ... io.nosqlbench.nb.mql.cli.RangeCLI --metric ops_total --window 5m
|
||||
*/
|
||||
@Command(
|
||||
name = "range",
|
||||
description = "Retrieve time-series data over a time window",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class RangeCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h, 30s)"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window (e.g., last 5m)"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Option(
|
||||
names = {"--start"},
|
||||
description = "Start timestamp in epoch milliseconds"
|
||||
)
|
||||
private Long start;
|
||||
|
||||
@Option(
|
||||
names = {"--end"},
|
||||
description = "End timestamp in epoch milliseconds"
|
||||
)
|
||||
private Long end;
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new RangeCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = super.buildParams();
|
||||
|
||||
// Add time range parameters
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
} else if (start != null && end != null) {
|
||||
params.put("start", start);
|
||||
params.put("end", end);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new RangeCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.RateCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for rate query command.
|
||||
* Calculates per-second rate of change for counter metrics.
|
||||
*/
|
||||
@Command(
|
||||
name = "rate",
|
||||
description = "Calculate per-second rate of change for counters",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class RateCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h, 30s)"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window (e.g., last 5m)"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new RateCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = super.buildParams();
|
||||
|
||||
// Add time window parameter
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new RateCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.RatioCommand;
|
||||
import io.nosqlbench.nb.mql.format.OutputFormat;
|
||||
import io.nosqlbench.nb.mql.format.ResultFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/**
|
||||
* CLI for ratio query command.
|
||||
* Calculates ratio between two metrics (useful for error rates).
|
||||
*/
|
||||
@Command(
|
||||
name = "ratio",
|
||||
description = "Calculate ratio between two metrics (numerator / denominator)",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class RatioCLI implements Callable<Integer> {
|
||||
|
||||
@Option(
|
||||
names = {"-d", "--db"},
|
||||
description = "Path to metrics database (default: logs/metrics.db)",
|
||||
defaultValue = "logs/metrics.db"
|
||||
)
|
||||
private Path databasePath;
|
||||
|
||||
@Option(
|
||||
names = {"-n", "--numerator"},
|
||||
description = "Numerator metric name",
|
||||
required = true
|
||||
)
|
||||
private String numerator;
|
||||
|
||||
@Option(
|
||||
names = {"--denominator"},
|
||||
description = "Denominator metric name",
|
||||
required = true
|
||||
)
|
||||
private String denominator;
|
||||
|
||||
@Option(
|
||||
names = {"-l", "--labels"},
|
||||
description = "Label filters (applied to both metrics)",
|
||||
split = ","
|
||||
)
|
||||
private Map<String, String> labels = new LinkedHashMap<>();
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h)"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Option(
|
||||
names = {"-f", "--format"},
|
||||
description = "Output format: table, json, jsonl, csv, tsv, markdown",
|
||||
defaultValue = "table"
|
||||
)
|
||||
private OutputFormat format = OutputFormat.TABLE;
|
||||
|
||||
@Option(
|
||||
names = {"--explain"},
|
||||
description = "Show the SQL query instead of executing"
|
||||
)
|
||||
private boolean explain;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
|
||||
RatioCommand command = new RatioCommand();
|
||||
|
||||
Map<String, Object> params = new LinkedHashMap<>();
|
||||
params.put("numerator", numerator);
|
||||
params.put("denominator", denominator);
|
||||
|
||||
if (!labels.isEmpty()) {
|
||||
params.put("labels", labels);
|
||||
}
|
||||
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
}
|
||||
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
if (explain) {
|
||||
System.out.println("SQL Query:");
|
||||
System.out.println(result.executedSQL());
|
||||
return 0;
|
||||
}
|
||||
|
||||
ResultFormatter formatter = format.createFormatter();
|
||||
System.out.println(formatter.format(result));
|
||||
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error: " + e.getMessage());
|
||||
if (System.getenv("DEBUG") != null) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new RatioCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.SessionCommand;
|
||||
import io.nosqlbench.nb.mql.format.OutputFormat;
|
||||
import io.nosqlbench.nb.mql.format.ResultFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/**
|
||||
* CLI for session command.
|
||||
* Displays session time information.
|
||||
*/
|
||||
@Command(
|
||||
name = "session",
|
||||
description = "Display session time information and snapshot cadence",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class SessionCLI implements Callable<Integer> {
|
||||
|
||||
@Option(
|
||||
names = {"-d", "--db"},
|
||||
description = "Path to metrics database (default: logs/metrics.db)",
|
||||
defaultValue = "logs/metrics.db"
|
||||
)
|
||||
private Path databasePath;
|
||||
|
||||
@Option(
|
||||
names = {"-f", "--format"},
|
||||
description = "Output format: table, json, jsonl, csv, tsv, markdown",
|
||||
defaultValue = "table"
|
||||
)
|
||||
private OutputFormat format = OutputFormat.TABLE;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
|
||||
SessionCommand command = new SessionCommand();
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
ResultFormatter formatter = format.createFormatter();
|
||||
System.out.println(formatter.format(result));
|
||||
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error: " + e.getMessage());
|
||||
if (System.getenv("DEBUG") != null) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new SessionCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.SqlCommand;
|
||||
import io.nosqlbench.nb.mql.format.OutputFormat;
|
||||
import io.nosqlbench.nb.mql.format.ResultFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/**
|
||||
* CLI for raw SQL query execution.
|
||||
* Allows direct SQLite queries for advanced users.
|
||||
*/
|
||||
@Command(
|
||||
name = "sql",
|
||||
description = "Execute raw SQLite queries (read-only)",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class SqlCLI implements Callable<Integer> {
|
||||
|
||||
@Option(
|
||||
names = {"-d", "--db"},
|
||||
description = "Path to metrics database (default: logs/metrics.db)",
|
||||
defaultValue = "logs/metrics.db"
|
||||
)
|
||||
private Path databasePath;
|
||||
|
||||
@Option(
|
||||
names = {"-q", "--query"},
|
||||
description = "SQL query to execute (SELECT, WITH, or PRAGMA only)",
|
||||
required = true
|
||||
)
|
||||
private String query;
|
||||
|
||||
@Option(
|
||||
names = {"-t", "--timeout"},
|
||||
description = "Query timeout in seconds (default: 30)",
|
||||
defaultValue = "30"
|
||||
)
|
||||
private int timeout;
|
||||
|
||||
@Option(
|
||||
names = {"-f", "--format"},
|
||||
description = "Output format: table, json, jsonl, csv, tsv, markdown",
|
||||
defaultValue = "table"
|
||||
)
|
||||
private OutputFormat format = OutputFormat.TABLE;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
Map<String, Object> params = new LinkedHashMap<>();
|
||||
params.put("query", query);
|
||||
params.put("timeout", timeout);
|
||||
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
ResultFormatter formatter = format.createFormatter();
|
||||
System.out.println(formatter.format(result));
|
||||
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error: " + e.getMessage());
|
||||
if (System.getenv("DEBUG") != null) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new SqlCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.SummaryCommand;
|
||||
import io.nosqlbench.nb.mql.format.OutputFormat;
|
||||
import io.nosqlbench.nb.mql.format.ResultFormatter;
|
||||
import io.nosqlbench.nb.mql.format.SummaryMarkdownFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
/**
|
||||
* CLI for summary command.
|
||||
* Displays database overview and metrics inventory.
|
||||
*/
|
||||
@Command(
|
||||
name = "summary",
|
||||
description = "Display overview of metrics in the database",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class SummaryCLI implements Callable<Integer> {
|
||||
|
||||
@Option(
|
||||
names = {"-d", "--db"},
|
||||
description = "Path to metrics database (default: logs/metrics.db)",
|
||||
defaultValue = "logs/metrics.db"
|
||||
)
|
||||
private Path databasePath;
|
||||
|
||||
@Option(
|
||||
names = {"-f", "--format"},
|
||||
description = "Output format: table, json, jsonl, csv, tsv, markdown",
|
||||
defaultValue = "table"
|
||||
)
|
||||
private OutputFormat format = OutputFormat.TABLE;
|
||||
|
||||
@Override
|
||||
public Integer call() throws Exception {
|
||||
try (Connection conn = MetricsDatabaseReader.connect(databasePath)) {
|
||||
SummaryCommand command = new SummaryCommand();
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
// Use custom markdown formatter for summary command
|
||||
ResultFormatter formatter;
|
||||
if (format == OutputFormat.MARKDOWN) {
|
||||
formatter = new SummaryMarkdownFormatter();
|
||||
} else {
|
||||
formatter = format.createFormatter();
|
||||
}
|
||||
|
||||
System.out.println(formatter.format(result));
|
||||
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error: " + e.getMessage());
|
||||
if (System.getenv("DEBUG") != null) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new SummaryCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.cli;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.TopKCommand;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import picocli.CommandLine;
|
||||
import picocli.CommandLine.Command;
|
||||
import picocli.CommandLine.Option;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* CLI for topk query command.
|
||||
* Finds top N label sets by metric value.
|
||||
*/
|
||||
@Command(
|
||||
name = "topk",
|
||||
description = "Find top N label sets by metric value",
|
||||
mixinStandardHelpOptions = true
|
||||
)
|
||||
public class TopKCLI extends AbstractQueryCommandCLI {
|
||||
|
||||
@Option(
|
||||
names = {"-n", "--top"},
|
||||
description = "Number of top results to return",
|
||||
required = true
|
||||
)
|
||||
private int n;
|
||||
|
||||
@Option(
|
||||
names = {"-w", "--window"},
|
||||
description = "Time window (e.g., 5m, 1h) - optional, defaults to latest snapshot"
|
||||
)
|
||||
private String window;
|
||||
|
||||
@Option(
|
||||
names = {"--last"},
|
||||
description = "Alternative to --window"
|
||||
)
|
||||
private String last;
|
||||
|
||||
@Override
|
||||
protected MetricsQueryCommand createCommand() {
|
||||
return new TopKCommand();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, Object> buildParams() {
|
||||
Map<String, Object> params = super.buildParams();
|
||||
|
||||
params.put("n", n);
|
||||
|
||||
if (window != null) {
|
||||
params.put("window", window);
|
||||
} else if (last != null) {
|
||||
params.put("last", last);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method for standalone execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new TopKCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.AggregateCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Aggregate command - Perform aggregations (sum, avg, max, min, count) on metrics.
|
||||
*
|
||||
* Usage: agg --metric <name> --func <sum|avg|max|min|count> [--by <label,...>] [--window <duration>] [--labels key=val,...]
|
||||
*
|
||||
* Aggregates metric values using SQL aggregate functions, optionally grouping by label dimensions.
|
||||
* This is the PromQL aggregation operator (sum by, avg by, etc.).
|
||||
*/
|
||||
public class AggregateCommand implements MetricsQueryCommand {
|
||||
|
||||
private static final Set<String> VALID_FUNCTIONS = Set.of("sum", "avg", "max", "min", "count");
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "agg";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Perform aggregations (sum, avg, max, min, count) on metrics";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
String function = ((String) params.get("function")).toLowerCase();
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> groupByLabels = (List<String>) params.getOrDefault("by", List.of());
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range (optional - defaults to latest snapshot)
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last");
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("last")) {
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
// Build the query
|
||||
String sql = buildQuery(metric, function, groupByLabels, labels, startTimestampMs, endTimestampMs, useLatestOnly);
|
||||
|
||||
// Build columns dynamically based on grouping
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.addAll(groupByLabels);
|
||||
columns.add(function);
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
|
||||
if (!useLatestOnly && startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
|
||||
// Add grouped label values
|
||||
for (String labelName : groupByLabels) {
|
||||
row.put(labelName, rs.getString(labelName));
|
||||
}
|
||||
|
||||
// Add aggregate result
|
||||
row.put(function, rs.getDouble(function));
|
||||
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
if (!params.containsKey("function")) {
|
||||
throw new InvalidQueryException("Missing required parameter: function (sum, avg, max, min, count)");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
Object function = params.get("function");
|
||||
if (!(function instanceof String)) {
|
||||
throw new InvalidQueryException("Parameter 'function' must be a string");
|
||||
}
|
||||
|
||||
String func = ((String) function).toLowerCase();
|
||||
if (!VALID_FUNCTIONS.contains(func)) {
|
||||
throw new InvalidQueryException(
|
||||
"Invalid function: " + function + "\n" +
|
||||
"Valid functions: " + String.join(", ", VALID_FUNCTIONS)
|
||||
);
|
||||
}
|
||||
|
||||
// Validate 'by' parameter if present
|
||||
if (params.containsKey("by")) {
|
||||
Object by = params.get("by");
|
||||
if (!(by instanceof List)) {
|
||||
throw new InvalidQueryException("Parameter 'by' must be a List<String>");
|
||||
}
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Sum all requests
|
||||
agg --metric api_requests_total --func sum
|
||||
|
||||
# Average by endpoint
|
||||
agg --metric api_requests_total --func avg --by endpoint
|
||||
|
||||
# Max latency by operation
|
||||
agg --metric operation_latency --func max --by operation
|
||||
|
||||
# Count by status code and method
|
||||
agg --metric api_requests_total --func count --by status,method
|
||||
|
||||
# Sum over last 5 minutes
|
||||
agg --metric requests_total --func sum --window 5m
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(String metric, String function, List<String> groupByLabels,
|
||||
Map<String, String> labelFilters, Long startTs, Long endTs, boolean useLatestOnly) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("WITH labeled_values AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE);
|
||||
|
||||
// Add individual label columns for grouping
|
||||
if (!groupByLabels.isEmpty()) {
|
||||
sql.append(",\n");
|
||||
for (int i = 0; i < groupByLabels.size(); i++) {
|
||||
sql.append(" lk").append(i).append(".").append(MetricsSchema.COL_LK_NAME).append(" AS label_key_").append(i).append(",\n");
|
||||
sql.append(" lv").append(i).append(".").append(MetricsSchema.COL_LV_VALUE).append(" AS label_value_").append(i);
|
||||
if (i < groupByLabels.size() - 1) {
|
||||
sql.append(",\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
sql.append("\n");
|
||||
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
|
||||
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
|
||||
if (useLatestOnly) {
|
||||
sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ")
|
||||
.append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n");
|
||||
}
|
||||
|
||||
// Join with label tables for grouping
|
||||
for (int i = 0; i < groupByLabels.size(); i++) {
|
||||
String labelName = groupByLabels.get(i);
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm").append(i)
|
||||
.append(" ON lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_SET_ID)
|
||||
.append(" = sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk").append(i)
|
||||
.append(" ON lk").append(i).append(".").append(MetricsSchema.COL_LK_ID)
|
||||
.append(" = lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_KEY_ID)
|
||||
.append(" AND lk").append(i).append(".").append(MetricsSchema.COL_LK_NAME).append(" = '").append(labelName).append("'\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv").append(i)
|
||||
.append(" ON lv").append(i).append(".").append(MetricsSchema.COL_LV_ID)
|
||||
.append(" = lsm").append(i).append(".").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
}
|
||||
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
|
||||
if (useLatestOnly) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n");
|
||||
} else if (startTs != null) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
int filterIdx = 0;
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
filterIdx++;
|
||||
}
|
||||
|
||||
sql.append(")\n");
|
||||
|
||||
// Aggregation query
|
||||
sql.append("SELECT\n");
|
||||
for (int i = 0; i < groupByLabels.size(); i++) {
|
||||
sql.append(" label_value_").append(i).append(" AS ").append(groupByLabels.get(i)).append(",\n");
|
||||
}
|
||||
sql.append(" ").append(function.toUpperCase()).append("(value) AS ").append(function).append("\n");
|
||||
sql.append("FROM labeled_values\n");
|
||||
|
||||
if (!groupByLabels.isEmpty()) {
|
||||
sql.append("GROUP BY ");
|
||||
for (int i = 0; i < groupByLabels.size(); i++) {
|
||||
sql.append("label_value_").append(i);
|
||||
if (i < groupByLabels.size() - 1) {
|
||||
sql.append(", ");
|
||||
}
|
||||
}
|
||||
sql.append("\n");
|
||||
}
|
||||
|
||||
sql.append("ORDER BY ").append(function);
|
||||
if (function.equals("count") || function.equals("sum")) {
|
||||
sql.append(" DESC"); // Descending for counts and sums
|
||||
}
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new AggregateCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.IncreaseCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Increase command - Calculate total increase in counter value over a time window.
|
||||
*
|
||||
* Usage: increase --metric <counter_metric> --window <duration> [--labels key=val,...]
|
||||
*
|
||||
* Returns the total increase (max - min) for each label set within the time window.
|
||||
* This is the standard PromQL increase() function.
|
||||
*
|
||||
* Important: Works only with counter metrics (monotonically increasing values).
|
||||
*/
|
||||
public class IncreaseCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "increase";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Calculate total increase in counter over time window";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("last")) {
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
// Build the query
|
||||
String sql = buildQuery(labels, startTimestampMs != null);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("increase");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
|
||||
if (startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("increase", rs.getDouble("increase"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
// Check that either window OR last is provided
|
||||
boolean hasWindow = params.containsKey("window");
|
||||
boolean hasLast = params.containsKey("last");
|
||||
|
||||
if (!hasWindow && !hasLast) {
|
||||
throw new InvalidQueryException(
|
||||
"Must specify time window using:\n" +
|
||||
" --window <duration> (e.g., 5m, 1h)\n" +
|
||||
" --last <duration> (e.g., 5m, 1h)"
|
||||
);
|
||||
}
|
||||
|
||||
if (hasWindow && hasLast) {
|
||||
throw new InvalidQueryException(
|
||||
"Cannot specify both --window and --last. Choose one."
|
||||
);
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Total requests in last hour
|
||||
increase --metric api_requests_total --window 1h
|
||||
|
||||
# Total errors in last 5 minutes
|
||||
increase --metric api_requests_total --window 5m --labels status=500
|
||||
|
||||
# Operations completed in last day
|
||||
increase --metric activity_ops_total --window 1d
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(Map<String, String> labelFilters, boolean hasTimeRange) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("WITH timeseries AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
|
||||
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
|
||||
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
|
||||
if (hasTimeRange) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
|
||||
sql.append(")\n");
|
||||
|
||||
// Calculate increase as MAX - MIN within window for each label set
|
||||
sql.append("SELECT\n");
|
||||
sql.append(" MAX(value) - MIN(value) AS increase,\n");
|
||||
sql.append(" labels\n");
|
||||
sql.append("FROM timeseries\n");
|
||||
sql.append("GROUP BY label_set_id, labels\n");
|
||||
sql.append("HAVING MAX(value) > MIN(value)\n"); // Only show increases > 0
|
||||
sql.append("ORDER BY labels");
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution of increase queries.
|
||||
* Delegates to IncreaseCLI for picocli argument parsing.
|
||||
*
|
||||
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.IncreaseCommand --metric ops_total --window 1h
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new IncreaseCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.InstantCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Instant query command - retrieves the most recent value for metrics.
|
||||
*
|
||||
* Usage: instant --metric <metric_name> [--labels key=val,...]
|
||||
*
|
||||
* Returns the latest snapshot value for each unique label set.
|
||||
*/
|
||||
public class InstantCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "instant";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Get the most recent value for a metric";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Build the query
|
||||
String sql = buildQuery(labels);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("timestamp");
|
||||
columns.add("value");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("timestamp", rs.getTimestamp("timestamp"));
|
||||
row.put("value", rs.getDouble("value"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Get latest value for all label sets
|
||||
instant --metric activity_ops_total
|
||||
|
||||
# Get latest value for specific labels
|
||||
instant --metric activity_ops_total --labels activity=read,host=server1
|
||||
|
||||
# Get latest latency
|
||||
instant --metric operation_latency_ms
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(Map<String, String> labelFilters) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("WITH latest_snapshot AS (\n");
|
||||
sql.append(" SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append("\n");
|
||||
sql.append("),\n");
|
||||
|
||||
sql.append("labeled_samples AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME).append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels,\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.").append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
sql.append(" CROSS JOIN latest_snapshot\n");
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest_snapshot.max_ts\n");
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.").append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.").append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.").append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
|
||||
sql.append(")\n");
|
||||
|
||||
sql.append("SELECT\n");
|
||||
sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n");
|
||||
sql.append(" value,\n");
|
||||
sql.append(" labels\n");
|
||||
sql.append("FROM labeled_samples\n");
|
||||
sql.append("ORDER BY labels");
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution of instant queries.
|
||||
* Delegates to InstantCLI for picocli argument parsing.
|
||||
*
|
||||
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.InstantCommand --metric ops_total
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new InstantCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.QuantileCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Quantile command - Extract percentile values from timer/summary metrics.
|
||||
*
|
||||
* Usage: quantile --metric <timer_metric> --p <0.0-1.0> [--window <duration>] [--labels key=val,...]
|
||||
*
|
||||
* Retrieves specific quantile values (e.g., p50, p95, p99) from timer or summary metrics.
|
||||
* Useful for latency analysis and SLA monitoring.
|
||||
*/
|
||||
public class QuantileCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "quantile";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Extract percentile values from timer/summary metrics";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
Double quantile = (Double) params.get("quantile");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range (optional for quantile)
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("last")) {
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
// Build the query
|
||||
String sql = buildQuery(labels, quantile, startTimestampMs != null);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("timestamp");
|
||||
columns.add("quantile");
|
||||
columns.add("quantile_value");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
ps.setDouble(paramIndex++, quantile);
|
||||
|
||||
if (startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("timestamp", rs.getTimestamp("timestamp"));
|
||||
row.put("quantile", rs.getDouble("quantile"));
|
||||
row.put("quantile_value", rs.getDouble("quantile_value"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
if (!params.containsKey("quantile")) {
|
||||
throw new InvalidQueryException("Missing required parameter: quantile (e.g., 0.95 for p95)");
|
||||
}
|
||||
|
||||
Object quantile = params.get("quantile");
|
||||
if (!(quantile instanceof Double)) {
|
||||
throw new InvalidQueryException("Parameter 'quantile' must be a number between 0.0 and 1.0");
|
||||
}
|
||||
|
||||
double q = (Double) quantile;
|
||||
if (q < 0.0 || q > 1.0) {
|
||||
throw new InvalidQueryException("Parameter 'quantile' must be between 0.0 and 1.0, got: " + q);
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Get p95 latency for all operations
|
||||
quantile --metric operation_latency_ms --p 0.95
|
||||
|
||||
# Get p99 for specific operation
|
||||
quantile --metric api_latency_ms --p 0.99 --labels operation=select
|
||||
|
||||
# p50 (median) over last 5 minutes
|
||||
quantile --metric latency_ms --p 0.50 --window 5m
|
||||
|
||||
# Common percentiles
|
||||
--p 0.50 # p50 (median)
|
||||
--p 0.95 # p95
|
||||
--p 0.99 # p99
|
||||
--p 0.999 # p999
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(Map<String, String> labelFilters, double quantile, boolean hasTimeRange) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("SELECT\n");
|
||||
sql.append(" datetime(sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" / 1000, 'unixepoch') AS timestamp,\n");
|
||||
sql.append(" sq.").append(MetricsSchema.COL_SQ_QUANTILE).append(",\n");
|
||||
sql.append(" sq.").append(MetricsSchema.COL_SQ_QUANTILE_VALUE).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
|
||||
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
|
||||
sql.append("FROM ").append(MetricsSchema.TABLE_SAMPLE_QUANTILE).append(" sq\n");
|
||||
sql.append("JOIN ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv ON sq.")
|
||||
.append(MetricsSchema.COL_SQ_SAMPLE_VALUE_ID).append(" = sv.").append(MetricsSchema.COL_SV_ID).append("\n");
|
||||
sql.append("JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sv.")
|
||||
.append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append(" = sn.").append(MetricsSchema.COL_SN_ID).append("\n");
|
||||
sql.append("").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
sql.append("WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
sql.append(" AND sq.").append(MetricsSchema.COL_SQ_QUANTILE).append(" = ?\n");
|
||||
|
||||
if (hasTimeRange) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append("GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append(", sq.")
|
||||
.append(MetricsSchema.COL_SQ_QUANTILE).append(", sq.").append(MetricsSchema.COL_SQ_QUANTILE_VALUE).append("\n");
|
||||
sql.append("ORDER BY sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(", labels");
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution of quantile queries.
|
||||
* Delegates to QuantileCLI for picocli argument parsing.
|
||||
*
|
||||
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.QuantileCommand --metric latency_ms --p 0.95
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new QuantileCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.RangeCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Range query command - retrieves time-series data over a time window.
|
||||
*
|
||||
* Usage: range --metric <metric_name> --window <duration> [--labels key=val,...]
|
||||
* range --metric <metric_name> --start <timestamp> --end <timestamp> [--labels key=val,...]
|
||||
*
|
||||
* Returns all data points within the specified time range for each label set.
|
||||
*/
|
||||
public class RangeCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "range";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Retrieve time-series data over a time window";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
// Use --window to query last N duration from now
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("start") && params.containsKey("end")) {
|
||||
// Use explicit start/end timestamps (epoch millis)
|
||||
startTimestampMs = (Long) params.get("start");
|
||||
endTimestampMs = (Long) params.get("end");
|
||||
} else if (params.containsKey("last")) {
|
||||
// Alternative: --last 5m
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
// Build the query
|
||||
String sql = buildQuery(labels, startTimestampMs != null);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("timestamp");
|
||||
columns.add("value");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
|
||||
if (startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("timestamp", rs.getTimestamp("timestamp"));
|
||||
row.put("value", rs.getDouble("value"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
// Check that either window OR (start AND end) OR last is provided
|
||||
boolean hasWindow = params.containsKey("window");
|
||||
boolean hasStartEnd = params.containsKey("start") && params.containsKey("end");
|
||||
boolean hasLast = params.containsKey("last");
|
||||
|
||||
if (!hasWindow && !hasStartEnd && !hasLast) {
|
||||
throw new InvalidQueryException(
|
||||
"Must specify time range using one of:\n" +
|
||||
" --window <duration> (e.g., 5m, 1h)\n" +
|
||||
" --last <duration> (e.g., 5m, 1h)\n" +
|
||||
" --start <ms> --end <ms> (epoch milliseconds)"
|
||||
);
|
||||
}
|
||||
|
||||
if ((hasWindow && hasStartEnd) || (hasWindow && hasLast) || (hasStartEnd && hasLast)) {
|
||||
throw new InvalidQueryException(
|
||||
"Cannot specify multiple time range methods. Choose one of: --window, --last, or --start/--end"
|
||||
);
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Get last 5 minutes of data
|
||||
range --metric activity_ops_total --window 5m
|
||||
|
||||
# Alternative syntax
|
||||
range --metric activity_ops_total --last 5m
|
||||
|
||||
# Get data between specific timestamps
|
||||
range --metric activity_ops_total --start 1729681200000 --end 1729681500000
|
||||
|
||||
# With label filters
|
||||
range --metric activity_ops_total --window 10m --labels activity=read
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(Map<String, String> labelFilters, boolean hasTimeRange) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("WITH labeled_samples AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
|
||||
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels,\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
|
||||
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
|
||||
if (hasTimeRange) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
|
||||
sql.append(")\n");
|
||||
|
||||
sql.append("SELECT\n");
|
||||
sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n");
|
||||
sql.append(" value,\n");
|
||||
sql.append(" labels\n");
|
||||
sql.append("FROM labeled_samples\n");
|
||||
sql.append("ORDER BY timestamp_ms, labels");
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution of range queries.
|
||||
* Delegates to RangeCLI for picocli argument parsing.
|
||||
*
|
||||
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.RangeCommand --metric ops_total --window 5m
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new RangeCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.RateCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Rate command - Calculate per-second rate of change for counter metrics.
|
||||
*
|
||||
* Usage: rate --metric <counter_metric> --window <duration> [--labels key=val,...]
|
||||
*
|
||||
* Calculates the per-second rate by taking the difference between consecutive data points
|
||||
* and dividing by the time interval between them. This is the standard PromQL rate() function.
|
||||
*
|
||||
* Important: Works only with counter metrics (monotonically increasing values).
|
||||
*/
|
||||
public class RateCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "rate";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Calculate per-second rate of change for counters";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("last")) {
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
// Build the query
|
||||
String sql = buildQuery(labels, startTimestampMs != null);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("timestamp");
|
||||
columns.add("rate_per_sec");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
|
||||
if (startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("timestamp", rs.getTimestamp("timestamp"));
|
||||
row.put("rate_per_sec", rs.getDouble("rate_per_sec"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
// Check that either window OR last is provided (rate needs a time context)
|
||||
boolean hasWindow = params.containsKey("window");
|
||||
boolean hasLast = params.containsKey("last");
|
||||
|
||||
if (!hasWindow && !hasLast) {
|
||||
throw new InvalidQueryException(
|
||||
"Must specify time window using:\n" +
|
||||
" --window <duration> (e.g., 5m, 1h)\n" +
|
||||
" --last <duration> (e.g., 5m, 1h)"
|
||||
);
|
||||
}
|
||||
|
||||
if (hasWindow && hasLast) {
|
||||
throw new InvalidQueryException(
|
||||
"Cannot specify both --window and --last. Choose one."
|
||||
);
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Calculate request rate over last 5 minutes
|
||||
rate --metric api_requests_total --window 5m
|
||||
|
||||
# Alternative syntax
|
||||
rate --metric api_requests_total --last 5m
|
||||
|
||||
# With label filters
|
||||
rate --metric api_requests_total --window 5m --labels method=GET,status=200
|
||||
|
||||
# Longer window for smoother rate
|
||||
rate --metric activity_ops_total --window 1h
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(Map<String, String> labelFilters, boolean hasTimeRange) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
// Use window functions to calculate rate between consecutive points
|
||||
sql.append("WITH timeseries AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
|
||||
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
|
||||
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
|
||||
if (hasTimeRange) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
|
||||
sql.append(" ORDER BY sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(", sv.")
|
||||
.append(MetricsSchema.COL_SV_TIMESTAMP_MS).append("\n");
|
||||
sql.append("),\n");
|
||||
|
||||
// Calculate rates using LAG window function
|
||||
sql.append("rates AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" timestamp_ms,\n");
|
||||
sql.append(" label_set_id,\n");
|
||||
sql.append(" labels,\n");
|
||||
sql.append(" value,\n");
|
||||
sql.append(" LAG(value) OVER (PARTITION BY label_set_id ORDER BY timestamp_ms) AS prev_value,\n");
|
||||
sql.append(" LAG(timestamp_ms) OVER (PARTITION BY label_set_id ORDER BY timestamp_ms) AS prev_timestamp\n");
|
||||
sql.append(" FROM timeseries\n");
|
||||
sql.append(")\n");
|
||||
|
||||
sql.append("SELECT\n");
|
||||
sql.append(" datetime(timestamp_ms / 1000, 'unixepoch') AS timestamp,\n");
|
||||
sql.append(" CASE\n");
|
||||
sql.append(" WHEN prev_value IS NULL OR prev_timestamp IS NULL THEN NULL\n");
|
||||
sql.append(" WHEN timestamp_ms = prev_timestamp THEN 0.0\n");
|
||||
sql.append(" ELSE (value - prev_value) / ((timestamp_ms - prev_timestamp) / 1000.0)\n");
|
||||
sql.append(" END AS rate_per_sec,\n");
|
||||
sql.append(" labels\n");
|
||||
sql.append("FROM rates\n");
|
||||
sql.append("WHERE prev_value IS NOT NULL\n"); // Skip first point (no previous value)
|
||||
sql.append("ORDER BY timestamp_ms, labels");
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution of rate queries.
|
||||
* Delegates to RateCLI for picocli argument parsing.
|
||||
*
|
||||
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.RateCommand --metric ops_total --window 5m
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new RateCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.RatioCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Ratio command - Calculate ratio between two metrics (e.g., error rate).
|
||||
*
|
||||
* Usage: ratio --numerator <metric> --denominator <metric> [--window <duration>] [--labels key=val,...]
|
||||
*
|
||||
* Divides numerator metric by denominator metric for matched label sets.
|
||||
* Useful for calculating error rates, cache hit ratios, etc.
|
||||
*/
|
||||
public class RatioCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "ratio";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Calculate ratio between two metrics (numerator / denominator)";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String numerator = (String) params.get("numerator");
|
||||
String denominator = (String) params.get("denominator");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range (optional - defaults to latest)
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last");
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("last")) {
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
String sql = buildQuery(numerator, denominator, labels, startTimestampMs, endTimestampMs, useLatestOnly);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
if (!useLatestOnly) {
|
||||
columns.add("timestamp");
|
||||
}
|
||||
columns.add("ratio");
|
||||
columns.add("numerator_value");
|
||||
columns.add("denominator_value");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, numerator);
|
||||
ps.setString(paramIndex++, denominator);
|
||||
|
||||
if (!useLatestOnly && startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters (twice - once for each metric)
|
||||
for (int i = 0; i < 2; i++) {
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
if (!useLatestOnly) {
|
||||
row.put("timestamp", rs.getTimestamp("timestamp"));
|
||||
}
|
||||
row.put("ratio", rs.getDouble("ratio"));
|
||||
row.put("numerator_value", rs.getDouble("numerator_value"));
|
||||
row.put("denominator_value", rs.getDouble("denominator_value"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("numerator")) {
|
||||
throw new InvalidQueryException("Missing required parameter: numerator");
|
||||
}
|
||||
|
||||
if (!params.containsKey("denominator")) {
|
||||
throw new InvalidQueryException("Missing required parameter: denominator");
|
||||
}
|
||||
|
||||
Object numerator = params.get("numerator");
|
||||
if (!(numerator instanceof String) || ((String) numerator).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'numerator' must be a non-empty string");
|
||||
}
|
||||
|
||||
Object denominator = params.get("denominator");
|
||||
if (!(denominator instanceof String) || ((String) denominator).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'denominator' must be a non-empty string");
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Error rate (errors / total requests)
|
||||
ratio --numerator errors_total --denominator requests_total
|
||||
|
||||
# Cache hit rate
|
||||
ratio --numerator cache_hits --denominator cache_total
|
||||
|
||||
# Success rate over time
|
||||
ratio --numerator success_requests --denominator total_requests --window 5m
|
||||
|
||||
# With label filters
|
||||
ratio --numerator errors --denominator requests --labels endpoint=/api/users
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(String numerator, String denominator, Map<String, String> labelFilters,
|
||||
Long startTs, Long endTs, boolean useLatestOnly) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
// Build CTEs for numerator and denominator
|
||||
sql.append("WITH num_data AS (\n");
|
||||
sql.append(buildMetricCTE(numerator, "num", labelFilters, startTs, endTs, useLatestOnly));
|
||||
sql.append("),\n");
|
||||
sql.append("den_data AS (\n");
|
||||
sql.append(buildMetricCTE(denominator, "den", labelFilters, startTs, endTs, useLatestOnly));
|
||||
sql.append(")\n");
|
||||
|
||||
// Join and calculate ratio
|
||||
sql.append("SELECT\n");
|
||||
if (!useLatestOnly) {
|
||||
sql.append(" num_data.timestamp,\n");
|
||||
}
|
||||
sql.append(" CASE\n");
|
||||
sql.append(" WHEN den_data.value = 0 THEN NULL\n");
|
||||
sql.append(" ELSE num_data.value / den_data.value\n");
|
||||
sql.append(" END AS ratio,\n");
|
||||
sql.append(" num_data.value AS numerator_value,\n");
|
||||
sql.append(" den_data.value AS denominator_value,\n");
|
||||
sql.append(" num_data.labels\n");
|
||||
sql.append("FROM num_data\n");
|
||||
sql.append("JOIN den_data ON num_data.label_set_id = den_data.label_set_id");
|
||||
|
||||
if (!useLatestOnly) {
|
||||
sql.append(" AND num_data.timestamp_ms = den_data.timestamp_ms");
|
||||
}
|
||||
|
||||
sql.append("\nORDER BY ");
|
||||
if (!useLatestOnly) {
|
||||
sql.append("num_data.timestamp_ms, ");
|
||||
}
|
||||
sql.append("num_data.labels");
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private String buildMetricCTE(String metric, String alias, Map<String, String> labelFilters,
|
||||
Long startTs, Long endTs, boolean useLatestOnly) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
|
||||
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
|
||||
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
|
||||
if (useLatestOnly) {
|
||||
sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ")
|
||||
.append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n");
|
||||
}
|
||||
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
|
||||
if (useLatestOnly) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n");
|
||||
} else if (startTs != null) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID);
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new RatioCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.SessionCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Session command - Display session time information.
|
||||
*
|
||||
* Usage: session
|
||||
*
|
||||
* Shows timing information about the metrics session including first/last snapshots,
|
||||
* duration, and snapshot cadence. Useful for understanding the data timeline.
|
||||
*/
|
||||
public class SessionCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "session";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Display session time information and snapshot cadence";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
String sql = """
|
||||
SELECT
|
||||
MIN(timestamp_ms) as first_snapshot_ms,
|
||||
MAX(timestamp_ms) as last_snapshot_ms,
|
||||
COUNT(DISTINCT timestamp_ms) as total_snapshots,
|
||||
COUNT(*) as total_samples
|
||||
FROM sample_value
|
||||
""";
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("first_snapshot");
|
||||
columns.add("last_snapshot");
|
||||
columns.add("duration");
|
||||
columns.add("total_snapshots");
|
||||
columns.add("total_samples");
|
||||
columns.add("avg_interval");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
|
||||
if (rs.next()) {
|
||||
long firstMs = rs.getLong("first_snapshot_ms");
|
||||
long lastMs = rs.getLong("last_snapshot_ms");
|
||||
int totalSnapshots = rs.getInt("total_snapshots");
|
||||
int totalSamples = rs.getInt("total_samples");
|
||||
|
||||
long durationMs = lastMs - firstMs;
|
||||
long avgIntervalMs = totalSnapshots > 1 ? durationMs / (totalSnapshots - 1) : 0;
|
||||
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("first_snapshot", new Timestamp(firstMs).toString());
|
||||
row.put("last_snapshot", new Timestamp(lastMs).toString());
|
||||
row.put("duration", formatDuration(durationMs));
|
||||
row.put("total_snapshots", totalSnapshots);
|
||||
row.put("total_samples", totalSamples);
|
||||
row.put("avg_interval", formatDuration(avgIntervalMs));
|
||||
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
// No parameters required for session command
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Show session information
|
||||
session
|
||||
|
||||
# JSON format for scripting
|
||||
session --format json
|
||||
|
||||
# Markdown for documentation
|
||||
session --format markdown
|
||||
""";
|
||||
}
|
||||
|
||||
private String formatDuration(long millis) {
|
||||
if (millis == 0) {
|
||||
return "0s";
|
||||
}
|
||||
|
||||
long seconds = millis / 1000;
|
||||
long minutes = seconds / 60;
|
||||
long hours = minutes / 60;
|
||||
long days = hours / 24;
|
||||
|
||||
if (days > 0) {
|
||||
return String.format("%dd %dh %dm", days, hours % 24, minutes % 60);
|
||||
} else if (hours > 0) {
|
||||
return String.format("%dh %dm %ds", hours, minutes % 60, seconds % 60);
|
||||
} else if (minutes > 0) {
|
||||
return String.format("%dm %ds", minutes, seconds % 60);
|
||||
} else {
|
||||
return String.format("%ds", seconds);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new SessionCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.SqlCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* SQL command - Execute raw SQLite queries against the metrics database.
|
||||
*
|
||||
* Usage: sql --query "SELECT * FROM metric_family"
|
||||
*
|
||||
* Allows advanced users to write custom SQL queries for exploration,
|
||||
* debugging, and analytics not covered by high-level commands.
|
||||
*
|
||||
* Safety:
|
||||
* - Connection is read-only (no modifications possible)
|
||||
* - Query timeout prevents runaway queries
|
||||
* - Full SQLite dialect and functions available
|
||||
*/
|
||||
public class SqlCommand implements MetricsQueryCommand {
|
||||
|
||||
private static final int DEFAULT_QUERY_TIMEOUT_SECONDS = 30;
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "sql";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Execute raw SQLite queries (read-only)";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String sql = (String) params.get("query");
|
||||
Integer timeoutSeconds = (Integer) params.getOrDefault("timeout", DEFAULT_QUERY_TIMEOUT_SECONDS);
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
ps.setQueryTimeout(timeoutSeconds);
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
// Extract column names from result set metadata
|
||||
ResultSetMetaData metadata = rs.getMetaData();
|
||||
int columnCount = metadata.getColumnCount();
|
||||
|
||||
for (int i = 1; i <= columnCount; i++) {
|
||||
columns.add(metadata.getColumnName(i));
|
||||
}
|
||||
|
||||
// Extract rows
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
for (int i = 1; i <= columnCount; i++) {
|
||||
String columnName = metadata.getColumnName(i);
|
||||
Object value = rs.getObject(i);
|
||||
row.put(columnName, value);
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("query")) {
|
||||
throw new InvalidQueryException("Missing required parameter: query");
|
||||
}
|
||||
|
||||
Object query = params.get("query");
|
||||
if (!(query instanceof String) || ((String) query).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'query' must be a non-empty SQL string");
|
||||
}
|
||||
|
||||
String sql = ((String) query).trim().toUpperCase();
|
||||
|
||||
// Basic validation: ensure it's a SELECT query (read-only)
|
||||
if (!sql.startsWith("SELECT") && !sql.startsWith("WITH") && !sql.startsWith("PRAGMA")) {
|
||||
throw new InvalidQueryException(
|
||||
"Only SELECT, WITH (CTE), and PRAGMA queries are allowed.\n" +
|
||||
"Database is read-only to prevent modifications."
|
||||
);
|
||||
}
|
||||
|
||||
// Block dangerous pragmas
|
||||
if (sql.contains("PRAGMA WRITABLE_SCHEMA") || sql.contains("PRAGMA CASE_SENSITIVE_LIKE")) {
|
||||
throw new InvalidQueryException("This PRAGMA is not allowed");
|
||||
}
|
||||
|
||||
// Validate timeout if present
|
||||
if (params.containsKey("timeout")) {
|
||||
Object timeout = params.get("timeout");
|
||||
if (!(timeout instanceof Integer) || (Integer) timeout <= 0) {
|
||||
throw new InvalidQueryException("Parameter 'timeout' must be a positive integer (seconds)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# List all metric families
|
||||
sql --query "SELECT name, type, unit FROM metric_family ORDER BY name"
|
||||
|
||||
# Get sample counts by metric
|
||||
sql --query "SELECT sn.sample, COUNT(*) as count FROM sample_value sv JOIN sample_name sn ON sv.sample_name_id = sn.id GROUP BY sn.sample"
|
||||
|
||||
# Custom time-series query
|
||||
sql --query "SELECT datetime(timestamp_ms/1000, 'unixepoch') as time, value FROM sample_value WHERE sample_name_id = 1 ORDER BY timestamp_ms"
|
||||
|
||||
# Schema inspection
|
||||
sql --query "PRAGMA table_info(sample_value)"
|
||||
|
||||
# With timeout
|
||||
sql --query "SELECT * FROM sample_value" --timeout 60
|
||||
""";
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution of SQL queries.
|
||||
* Delegates to SqlCLI for picocli argument parsing.
|
||||
*
|
||||
* Usage: java -cp ... io.nosqlbench.nb.mql.commands.SqlCommand --query "SELECT ..."
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new SqlCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.SummaryCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Summary command - Display comprehensive end-of-session metrics report.
|
||||
*
|
||||
* Usage: summary
|
||||
*
|
||||
* Provides an elaborate session summary focusing on NoSQLBench core metrics:
|
||||
* - Session timeline and duration
|
||||
* - Core operation metrics (cycles, result, tries, errors)
|
||||
* - Activity breakdowns
|
||||
* - Error analysis
|
||||
* - Performance indicators
|
||||
*
|
||||
* Designed to give a complete picture of the session at a glance.
|
||||
*/
|
||||
public class SummaryCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "summary";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Display comprehensive end-of-session metrics report";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Build comprehensive summary with multiple sections
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("section");
|
||||
columns.add("metric");
|
||||
columns.add("value");
|
||||
columns.add("details");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
// Section 1: Session Overview
|
||||
addSessionOverview(conn, rows);
|
||||
|
||||
// Section 2: Core Metrics Summary (cycles, result, tries, errors)
|
||||
addCoreMetricsSummary(conn, rows);
|
||||
|
||||
// Section 3: Activity Breakdown
|
||||
addActivityBreakdown(conn, rows);
|
||||
|
||||
// Section 4: Error Analysis
|
||||
addErrorAnalysis(conn, rows);
|
||||
|
||||
// Section 5: All Available Metrics
|
||||
addAllMetrics(conn, rows);
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
// Build a single SQL representing all queries (for reference)
|
||||
String combinedSql = "-- Multiple queries executed for comprehensive summary\n" +
|
||||
"-- See SummaryCommand.java for details";
|
||||
|
||||
return new QueryResult(columns, rows, combinedSql, executionTime);
|
||||
}
|
||||
|
||||
private void addSessionOverview(Connection conn, List<Map<String, Object>> rows) throws SQLException {
|
||||
String sql = """
|
||||
SELECT
|
||||
MIN(timestamp_ms) as first_ts,
|
||||
MAX(timestamp_ms) as last_ts,
|
||||
COUNT(DISTINCT timestamp_ms) as snapshots,
|
||||
COUNT(*) as total_samples
|
||||
FROM sample_value
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
|
||||
if (rs.next()) {
|
||||
long firstMs = rs.getLong("first_ts");
|
||||
long lastMs = rs.getLong("last_ts");
|
||||
int snapshots = rs.getInt("snapshots");
|
||||
int totalSamples = rs.getInt("total_samples");
|
||||
long durationMs = lastMs - firstMs;
|
||||
|
||||
rows.add(createRow("SESSION", "Time Range",
|
||||
new Timestamp(firstMs) + " to " + new Timestamp(lastMs),
|
||||
formatDuration(durationMs)));
|
||||
|
||||
rows.add(createRow("SESSION", "Snapshots", String.valueOf(snapshots),
|
||||
"Total samples: " + totalSamples));
|
||||
|
||||
if (snapshots > 1) {
|
||||
long avgInterval = durationMs / (snapshots - 1);
|
||||
rows.add(createRow("SESSION", "Avg Interval", formatDuration(avgInterval),
|
||||
"~" + (avgInterval / 1000) + "s cadence"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addCoreMetricsSummary(Connection conn, List<Map<String, Object>> rows) throws SQLException {
|
||||
// Query for core NoSQLBench metrics: cycles, result, tries, errors
|
||||
String[] coreMetrics = {"cycles", "result", "tries", "errors"};
|
||||
|
||||
for (String metricBase : coreMetrics) {
|
||||
String sql = """
|
||||
SELECT
|
||||
SUM(sv.value) as total_value,
|
||||
COUNT(DISTINCT sv.label_set_id) as label_sets
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
WHERE sn.sample LIKE ?
|
||||
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
ps.setString(1, "%" + metricBase + "%");
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
double total = rs.getDouble("total_value");
|
||||
int labelSets = rs.getInt("label_sets");
|
||||
|
||||
if (labelSets > 0) {
|
||||
rows.add(createRow("CORE METRICS", metricBase.toUpperCase(),
|
||||
String.format("%.0f", total),
|
||||
labelSets + " label set(s)"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate error rate if we have both result and errors
|
||||
calculateErrorRate(conn, rows);
|
||||
}
|
||||
|
||||
private void calculateErrorRate(Connection conn, List<Map<String, Object>> rows) throws SQLException {
|
||||
String sql = """
|
||||
SELECT
|
||||
SUM(CASE WHEN sn.sample LIKE '%result%' THEN sv.value ELSE 0 END) as results,
|
||||
SUM(CASE WHEN sn.sample LIKE '%error%' THEN sv.value ELSE 0 END) as errors
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
WHERE sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
|
||||
if (rs.next()) {
|
||||
double results = rs.getDouble("results");
|
||||
double errors = rs.getDouble("errors");
|
||||
|
||||
if (results > 0) {
|
||||
double errorRate = (errors / results) * 100.0;
|
||||
rows.add(createRow("CORE METRICS", "Error Rate",
|
||||
String.format("%.2f%%", errorRate),
|
||||
String.format("%.0f errors / %.0f results", errors, results)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addActivityBreakdown(Connection conn, List<Map<String, Object>> rows) throws SQLException {
|
||||
// Group by activity label if it exists
|
||||
String sql = """
|
||||
SELECT
|
||||
lv.value as activity,
|
||||
SUM(sv.value) as total
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
|
||||
JOIN label_key lk ON lsm.label_key_id = lk.id
|
||||
JOIN label_value lv ON lsm.label_value_id = lv.id
|
||||
WHERE lk.name = 'activity'
|
||||
AND sn.sample LIKE '%ops%'
|
||||
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
GROUP BY lv.value
|
||||
ORDER BY total DESC
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
|
||||
while (rs.next()) {
|
||||
String activity = rs.getString("activity");
|
||||
double total = rs.getDouble("total");
|
||||
|
||||
rows.add(createRow("ACTIVITY", activity,
|
||||
String.format("%.0f", total), "operations"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addErrorAnalysis(Connection conn, List<Map<String, Object>> rows) throws SQLException {
|
||||
String sql = """
|
||||
SELECT
|
||||
sn.sample,
|
||||
SUM(sv.value) as total_errors
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
WHERE sn.sample LIKE '%error%'
|
||||
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
GROUP BY sn.sample
|
||||
HAVING total_errors > 0
|
||||
ORDER BY total_errors DESC
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
|
||||
while (rs.next()) {
|
||||
String sample = rs.getString("sample");
|
||||
double errors = rs.getDouble("total_errors");
|
||||
|
||||
rows.add(createRow("ERRORS", sample,
|
||||
String.format("%.0f", errors), "total errors"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addAllMetrics(Connection conn, List<Map<String, Object>> rows) throws SQLException {
|
||||
String sql = """
|
||||
SELECT
|
||||
mf.name AS metric_name,
|
||||
mf.type AS metric_type,
|
||||
COUNT(DISTINCT sv.label_set_id) AS unique_label_sets,
|
||||
COUNT(*) AS total_samples,
|
||||
MIN(sv.value) AS min_value,
|
||||
MAX(sv.value) AS max_value
|
||||
FROM metric_family mf
|
||||
JOIN sample_name sn ON sn.metric_family_id = mf.id
|
||||
JOIN sample_value sv ON sv.sample_name_id = sn.id
|
||||
GROUP BY mf.id, mf.name, mf.type
|
||||
ORDER BY mf.name
|
||||
""";
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
|
||||
while (rs.next()) {
|
||||
String name = rs.getString("metric_name");
|
||||
String type = rs.getString("metric_type");
|
||||
int labelSets = rs.getInt("unique_label_sets");
|
||||
int samples = rs.getInt("total_samples");
|
||||
double min = rs.getDouble("min_value");
|
||||
double max = rs.getDouble("max_value");
|
||||
|
||||
String details = String.format("type=%s, samples=%d, label_sets=%d, range=[%.1f, %.1f]",
|
||||
type, samples, labelSets, min, max);
|
||||
|
||||
rows.add(createRow("ALL METRICS", name, "", details));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> createRow(String section, String metric, String value, String details) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("section", section);
|
||||
row.put("metric", metric);
|
||||
row.put("value", value);
|
||||
row.put("details", details);
|
||||
return row;
|
||||
}
|
||||
|
||||
private String formatDuration(long millis) {
|
||||
if (millis == 0) {
|
||||
return "0s";
|
||||
}
|
||||
|
||||
long seconds = millis / 1000;
|
||||
long minutes = seconds / 60;
|
||||
long hours = minutes / 60;
|
||||
long days = hours / 24;
|
||||
|
||||
if (days > 0) {
|
||||
return String.format("%dd %dh %dm", days, hours % 24, minutes % 60);
|
||||
} else if (hours > 0) {
|
||||
return String.format("%dh %dm %ds", hours, minutes % 60, seconds % 60);
|
||||
} else if (minutes > 0) {
|
||||
return String.format("%dm %ds", minutes, seconds % 60);
|
||||
} else {
|
||||
return String.format("%ds", seconds);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
// No parameters required for summary command
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Show comprehensive session summary
|
||||
summary
|
||||
|
||||
# JSON format for scripting
|
||||
summary --format json
|
||||
|
||||
# Markdown for reports
|
||||
summary --format markdown
|
||||
|
||||
# Use at end of session
|
||||
nb5 run.yaml && nb5 mql summary
|
||||
""";
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new SummaryCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.cli.TopKCLI;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.MetricsQueryCommand;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.query.TimeWindowParser;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsSchema;
|
||||
import picocli.CommandLine;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* TopK command - Find top N label sets by metric value.
|
||||
*
|
||||
* Usage: topk --metric <name> --n <count> [--window <duration>] [--labels key=val,...]
|
||||
*
|
||||
* Returns the N label sets with highest metric values.
|
||||
* This is the PromQL topk() function.
|
||||
*/
|
||||
public class TopKCommand implements MetricsQueryCommand {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "topk";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Find top N label sets by metric value";
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryResult execute(Connection conn, Map<String, Object> params)
|
||||
throws SQLException, InvalidQueryException {
|
||||
|
||||
validate(params);
|
||||
|
||||
String metric = (String) params.get("metric");
|
||||
Integer n = (Integer) params.get("n");
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, String> labels = (Map<String, String>) params.getOrDefault("labels", Map.of());
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// Determine time range (optional - defaults to latest)
|
||||
Long startTimestampMs = null;
|
||||
Long endTimestampMs = null;
|
||||
boolean useLatestOnly = !params.containsKey("window") && !params.containsKey("last");
|
||||
|
||||
if (params.containsKey("window")) {
|
||||
String window = (String) params.get("window");
|
||||
long windowMs = TimeWindowParser.parseToMillis(window);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
} else if (params.containsKey("last")) {
|
||||
String last = (String) params.get("last");
|
||||
long windowMs = TimeWindowParser.parseToMillis(last);
|
||||
endTimestampMs = getCurrentMaxTimestamp(conn);
|
||||
startTimestampMs = endTimestampMs - windowMs;
|
||||
}
|
||||
|
||||
String sql = buildQuery(labels, startTimestampMs, endTimestampMs, useLatestOnly, n);
|
||||
|
||||
List<String> columns = new ArrayList<>();
|
||||
columns.add("value");
|
||||
columns.add("labels");
|
||||
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql)) {
|
||||
int paramIndex = 1;
|
||||
ps.setString(paramIndex++, metric);
|
||||
|
||||
if (!useLatestOnly && startTimestampMs != null) {
|
||||
ps.setLong(paramIndex++, startTimestampMs);
|
||||
ps.setLong(paramIndex++, endTimestampMs);
|
||||
}
|
||||
|
||||
// Add label filter parameters
|
||||
for (Map.Entry<String, String> label : labels.entrySet()) {
|
||||
ps.setString(paramIndex++, label.getKey());
|
||||
ps.setString(paramIndex++, label.getValue());
|
||||
}
|
||||
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("value", rs.getDouble("value"));
|
||||
row.put("labels", rs.getString("labels"));
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long executionTime = System.currentTimeMillis() - startTime;
|
||||
|
||||
return new QueryResult(columns, rows, sql, executionTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void validate(Map<String, Object> params) throws InvalidQueryException {
|
||||
if (!params.containsKey("metric")) {
|
||||
throw new InvalidQueryException("Missing required parameter: metric");
|
||||
}
|
||||
|
||||
if (!params.containsKey("n")) {
|
||||
throw new InvalidQueryException("Missing required parameter: n (number of top results)");
|
||||
}
|
||||
|
||||
Object metric = params.get("metric");
|
||||
if (!(metric instanceof String) || ((String) metric).trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Parameter 'metric' must be a non-empty string");
|
||||
}
|
||||
|
||||
Object n = params.get("n");
|
||||
if (!(n instanceof Integer) || (Integer) n <= 0) {
|
||||
throw new InvalidQueryException("Parameter 'n' must be a positive integer");
|
||||
}
|
||||
|
||||
// Validate labels if present
|
||||
if (params.containsKey("labels")) {
|
||||
Object labels = params.get("labels");
|
||||
if (!(labels instanceof Map)) {
|
||||
throw new InvalidQueryException("Parameter 'labels' must be a Map<String, String>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsageExamples() {
|
||||
return """
|
||||
Examples:
|
||||
# Top 5 endpoints by request count
|
||||
topk --metric api_requests_total --n 5
|
||||
|
||||
# Top 3 slowest operations
|
||||
topk --metric operation_latency --n 3
|
||||
|
||||
# Top 10 in last hour
|
||||
topk --metric requests_total --n 10 --window 1h
|
||||
|
||||
# Top 5 errors
|
||||
topk --metric errors_total --n 5 --labels status=500
|
||||
""";
|
||||
}
|
||||
|
||||
private String buildQuery(Map<String, String> labelFilters, Long startTs, Long endTs,
|
||||
boolean useLatestOnly, int limit) {
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
sql.append("WITH latest_values AS (\n");
|
||||
sql.append(" SELECT\n");
|
||||
sql.append(" sv.").append(MetricsSchema.COL_SV_VALUE).append(",\n");
|
||||
sql.append(" GROUP_CONCAT(lk.").append(MetricsSchema.COL_LK_NAME)
|
||||
.append(" || '=' || lv.").append(MetricsSchema.COL_LV_VALUE).append(", ', ') AS labels\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_SAMPLE_VALUE).append(" sv\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_SAMPLE_NAME).append(" sn ON sn.")
|
||||
.append(MetricsSchema.COL_SN_ID).append(" = sv.").append(MetricsSchema.COL_SV_SAMPLE_NAME_ID).append("\n");
|
||||
sql.append(" ").append(MetricsSchema.joinAllLabels()).append("\n");
|
||||
|
||||
if (useLatestOnly) {
|
||||
sql.append(" CROSS JOIN (SELECT MAX(").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(") AS max_ts FROM ")
|
||||
.append(MetricsSchema.TABLE_SAMPLE_VALUE).append(") latest\n");
|
||||
}
|
||||
|
||||
sql.append(" WHERE sn.").append(MetricsSchema.COL_SN_SAMPLE).append(" = ?\n");
|
||||
|
||||
if (useLatestOnly) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" = latest.max_ts\n");
|
||||
} else if (startTs != null) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" >= ?\n");
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_TIMESTAMP_MS).append(" <= ?\n");
|
||||
}
|
||||
|
||||
// Add label filters
|
||||
for (String labelKey : labelFilters.keySet()) {
|
||||
sql.append(" AND sv.").append(MetricsSchema.COL_SV_LABEL_SET_ID).append(" IN (\n");
|
||||
sql.append(" SELECT lsm.").append(MetricsSchema.COL_LSM_LABEL_SET_ID).append("\n");
|
||||
sql.append(" FROM ").append(MetricsSchema.TABLE_LABEL_SET_MEMBERSHIP).append(" lsm\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_KEY).append(" lk ON lk.")
|
||||
.append(MetricsSchema.COL_LK_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_KEY_ID).append("\n");
|
||||
sql.append(" JOIN ").append(MetricsSchema.TABLE_LABEL_VALUE).append(" lv ON lv.")
|
||||
.append(MetricsSchema.COL_LV_ID).append(" = lsm.").append(MetricsSchema.COL_LSM_LABEL_VALUE_ID).append("\n");
|
||||
sql.append(" WHERE lk.").append(MetricsSchema.COL_LK_NAME).append(" = ? AND lv.")
|
||||
.append(MetricsSchema.COL_LV_VALUE).append(" = ?\n");
|
||||
sql.append(" )\n");
|
||||
}
|
||||
|
||||
sql.append(" GROUP BY sv.").append(MetricsSchema.COL_SV_ID).append("\n");
|
||||
sql.append(")\n");
|
||||
|
||||
sql.append("SELECT value, labels\n");
|
||||
sql.append("FROM latest_values\n");
|
||||
sql.append("ORDER BY value DESC\n");
|
||||
sql.append("LIMIT ").append(limit);
|
||||
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
private long getCurrentMaxTimestamp(Connection conn) throws SQLException {
|
||||
String sql = "SELECT MAX(" + MetricsSchema.COL_SV_TIMESTAMP_MS + ") FROM " +
|
||||
MetricsSchema.TABLE_SAMPLE_VALUE;
|
||||
try (PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ResultSet rs = ps.executeQuery()) {
|
||||
if (rs.next()) {
|
||||
return rs.getLong(1);
|
||||
}
|
||||
throw new SQLException("No data in database");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone main method for direct execution.
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exitCode = new CommandLine(new TopKCLI()).execute(args);
|
||||
System.exit(exitCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Formats query results as CSV for spreadsheet import.
|
||||
*/
|
||||
public class CsvFormatter implements ResultFormatter {
|
||||
|
||||
private final boolean includeHeaders;
|
||||
|
||||
public CsvFormatter() {
|
||||
this(true);
|
||||
}
|
||||
|
||||
public CsvFormatter(boolean includeHeaders) {
|
||||
this.includeHeaders = includeHeaders;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
StringWriter writer = new StringWriter();
|
||||
|
||||
try {
|
||||
CSVFormat format = includeHeaders
|
||||
? CSVFormat.DEFAULT.builder().setHeader(result.columns().toArray(new String[0])).build()
|
||||
: CSVFormat.DEFAULT;
|
||||
|
||||
try (CSVPrinter printer = new CSVPrinter(writer, format)) {
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
List<Object> values = new ArrayList<>();
|
||||
for (String column : result.columns()) {
|
||||
values.add(row.get(column));
|
||||
}
|
||||
printer.printRecord(values);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to format CSV", e);
|
||||
}
|
||||
|
||||
return writer.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "csv";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "csv";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Formats query results as JSON for programmatic consumption.
|
||||
*/
|
||||
public class JsonFormatter implements ResultFormatter {
|
||||
|
||||
private final Gson gson;
|
||||
private final boolean pretty;
|
||||
|
||||
public JsonFormatter() {
|
||||
this(false);
|
||||
}
|
||||
|
||||
public JsonFormatter(boolean pretty) {
|
||||
this.pretty = pretty;
|
||||
GsonBuilder builder = new GsonBuilder();
|
||||
if (pretty) {
|
||||
builder.setPrettyPrinting();
|
||||
}
|
||||
this.gson = builder.create();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
Map<String, Object> output = new LinkedHashMap<>();
|
||||
output.put("columns", result.columns());
|
||||
output.put("rows", result.rows());
|
||||
output.put("rowCount", result.rowCount());
|
||||
output.put("executionTimeMs", result.executionTimeMs());
|
||||
|
||||
return gson.toJson(output);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "json";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "json";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Formats query results as JSONL (JSON Lines) for streaming data processing.
|
||||
* Each row is output as a single JSON object on its own line, with no array wrapper.
|
||||
* This format is efficient for large result sets and compatible with streaming processors.
|
||||
*/
|
||||
public class JsonLinesFormatter implements ResultFormatter {
|
||||
|
||||
private final Gson gson;
|
||||
|
||||
public JsonLinesFormatter() {
|
||||
// Compact JSON (no pretty printing for JSONL)
|
||||
this.gson = new Gson();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Each row is a separate JSON object on its own line
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
sb.append(gson.toJson(row));
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "jsonl";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "jsonl";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Formats query results as GitHub-flavored Markdown tables.
|
||||
* Output can be directly copy-pasted into markdown documentation.
|
||||
*/
|
||||
public class MarkdownFormatter implements ResultFormatter {
|
||||
|
||||
private static final int DEFAULT_MAX_COLUMN_WIDTH = 50;
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
if (result.isEmpty()) {
|
||||
return "*No results found.*\n";
|
||||
}
|
||||
|
||||
List<String> columns = result.columns();
|
||||
List<Map<String, Object>> rows = result.rows();
|
||||
|
||||
// Calculate column widths
|
||||
Map<String, Integer> columnWidths = calculateColumnWidths(columns, rows);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Header
|
||||
sb.append("| ");
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String column = columns.get(i);
|
||||
sb.append(padRight(column, columnWidths.get(column)));
|
||||
if (i < columns.size() - 1) {
|
||||
sb.append(" | ");
|
||||
}
|
||||
}
|
||||
sb.append(" |\n");
|
||||
|
||||
// Separator row
|
||||
sb.append("|");
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String column = columns.get(i);
|
||||
sb.append("-".repeat(columnWidths.get(column) + 2)); // +2 for padding spaces
|
||||
if (i < columns.size() - 1) {
|
||||
sb.append("|");
|
||||
}
|
||||
}
|
||||
sb.append("|\n");
|
||||
|
||||
// Data rows
|
||||
for (Map<String, Object> row : rows) {
|
||||
sb.append("| ");
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String column = columns.get(i);
|
||||
Object value = row.get(column);
|
||||
String strValue = value != null ? escapeMarkdown(value.toString()) : "";
|
||||
|
||||
// Truncate if too long
|
||||
if (strValue.length() > DEFAULT_MAX_COLUMN_WIDTH) {
|
||||
strValue = strValue.substring(0, DEFAULT_MAX_COLUMN_WIDTH - 3) + "...";
|
||||
}
|
||||
|
||||
sb.append(padRight(strValue, columnWidths.get(column)));
|
||||
if (i < columns.size() - 1) {
|
||||
sb.append(" | ");
|
||||
}
|
||||
}
|
||||
sb.append(" |\n");
|
||||
}
|
||||
|
||||
// Footer with metadata
|
||||
sb.append("\n");
|
||||
sb.append("*").append(result.rowCount()).append(" row");
|
||||
if (result.rowCount() != 1) {
|
||||
sb.append("s");
|
||||
}
|
||||
sb.append(" (").append(result.executionTimeMs()).append("ms)*\n");
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "markdown";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "md";
|
||||
}
|
||||
|
||||
private Map<String, Integer> calculateColumnWidths(List<String> columns, List<Map<String, Object>> rows) {
|
||||
Map<String, Integer> widths = new java.util.LinkedHashMap<>();
|
||||
|
||||
// Initialize with header widths
|
||||
for (String column : columns) {
|
||||
widths.put(column, column.length());
|
||||
}
|
||||
|
||||
// Check data widths
|
||||
for (Map<String, Object> row : rows) {
|
||||
for (String column : columns) {
|
||||
Object value = row.get(column);
|
||||
String strValue = value != null ? value.toString() : "";
|
||||
int currentWidth = widths.get(column);
|
||||
int valueWidth = Math.min(strValue.length(), DEFAULT_MAX_COLUMN_WIDTH);
|
||||
widths.put(column, Math.max(currentWidth, valueWidth));
|
||||
}
|
||||
}
|
||||
|
||||
return widths;
|
||||
}
|
||||
|
||||
private String padRight(String str, int width) {
|
||||
if (str.length() >= width) {
|
||||
return str;
|
||||
}
|
||||
return str + " ".repeat(width - str.length());
|
||||
}
|
||||
|
||||
private String escapeMarkdown(String str) {
|
||||
// Escape pipe characters which would break the table
|
||||
return str.replace("|", "\\|");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
/**
|
||||
* Enum representing all supported output formats for query results.
|
||||
* Provides type safety and validation for format selection in CLI.
|
||||
*/
|
||||
public enum OutputFormat {
|
||||
/**
|
||||
* ASCII tables for console output (default).
|
||||
*/
|
||||
TABLE("table", "txt", "ASCII tables for console output") {
|
||||
@Override
|
||||
public ResultFormatter createFormatter() {
|
||||
return new TableFormatter();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Comma-separated values for spreadsheet import.
|
||||
*/
|
||||
CSV("csv", "csv", "Comma-separated values for Excel/Sheets") {
|
||||
@Override
|
||||
public ResultFormatter createFormatter() {
|
||||
return new CsvFormatter();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Tab-separated values for Unix text processing.
|
||||
*/
|
||||
TSV("tsv", "tsv", "Tab-separated values for awk/cut/paste") {
|
||||
@Override
|
||||
public ResultFormatter createFormatter() {
|
||||
return new TsvFormatter();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Pretty-printed JSON for programmatic use.
|
||||
*/
|
||||
JSON("json", "json", "Pretty-printed JSON with metadata") {
|
||||
@Override
|
||||
public ResultFormatter createFormatter() {
|
||||
return new JsonFormatter(true);
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* JSON Lines (one JSON object per line) for streaming.
|
||||
*/
|
||||
JSONL("jsonl", "jsonl", "JSON Lines for streaming processors") {
|
||||
@Override
|
||||
public ResultFormatter createFormatter() {
|
||||
return new JsonLinesFormatter();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* GitHub-flavored Markdown tables for documentation.
|
||||
*/
|
||||
MARKDOWN("markdown", "md", "Markdown tables for documentation") {
|
||||
@Override
|
||||
public ResultFormatter createFormatter() {
|
||||
return new MarkdownFormatter();
|
||||
}
|
||||
};
|
||||
|
||||
private final String name;
|
||||
private final String extension;
|
||||
private final String description;
|
||||
|
||||
OutputFormat(String name, String extension, String description) {
|
||||
this.name = name;
|
||||
this.extension = extension;
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a formatter instance for this format.
|
||||
* Each enum value knows how to create its own formatter.
|
||||
*
|
||||
* @return ResultFormatter instance
|
||||
*/
|
||||
public abstract ResultFormatter createFormatter();
|
||||
|
||||
/**
|
||||
* Get the format name as it appears in CLI arguments.
|
||||
*/
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file extension for this format.
|
||||
*/
|
||||
public String getExtension() {
|
||||
return extension;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a human-readable description.
|
||||
*/
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a format name string to an enum value.
|
||||
*
|
||||
* @param name Format name (case-insensitive)
|
||||
* @return The OutputFormat enum, or null if not found
|
||||
*/
|
||||
public static OutputFormat fromString(String name) {
|
||||
if (name == null || name.trim().isEmpty()) {
|
||||
return TABLE; // Default
|
||||
}
|
||||
|
||||
String normalized = name.toLowerCase().trim();
|
||||
for (OutputFormat format : values()) {
|
||||
if (format.name.equals(normalized)) {
|
||||
return format;
|
||||
}
|
||||
}
|
||||
return null; // Not found
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a comma-separated list of all format names.
|
||||
* Useful for CLI help text.
|
||||
*/
|
||||
public static String getAllFormatNames() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < values().length; i++) {
|
||||
sb.append(values()[i].name);
|
||||
if (i < values().length - 1) {
|
||||
sb.append(", ");
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
/**
|
||||
* Interface for formatting query results in different output formats.
|
||||
*/
|
||||
public interface ResultFormatter {
|
||||
|
||||
/**
|
||||
* Format a query result as a string.
|
||||
*
|
||||
* @param result The query result to format
|
||||
* @return Formatted string representation
|
||||
*/
|
||||
String format(QueryResult result);
|
||||
|
||||
/**
|
||||
* Get the name of this formatter (e.g., "table", "json", "csv").
|
||||
*
|
||||
* @return Formatter name
|
||||
*/
|
||||
String getName();
|
||||
|
||||
/**
|
||||
* Get the file extension for this format (e.g., "txt", "json", "csv").
|
||||
*
|
||||
* @return File extension without the dot
|
||||
*/
|
||||
String getFileExtension();
|
||||
}
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Custom markdown formatter for summary command output.
|
||||
* Creates a nicely structured report with tree views and sections.
|
||||
*/
|
||||
public class SummaryMarkdownFormatter implements ResultFormatter {
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
if (result.isEmpty()) {
|
||||
return "*No data available.*\n";
|
||||
}
|
||||
|
||||
StringBuilder md = new StringBuilder();
|
||||
|
||||
// Title
|
||||
md.append("# NoSQLBench Session Summary\n\n");
|
||||
|
||||
// Group rows by section
|
||||
Map<String, List<Map<String, Object>>> sections = new LinkedHashMap<>();
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String section = (String) row.get("section");
|
||||
sections.computeIfAbsent(section, k -> new ArrayList<>()).add(row);
|
||||
}
|
||||
|
||||
// Render each section
|
||||
for (Map.Entry<String, List<Map<String, Object>>> entry : sections.entrySet()) {
|
||||
String sectionName = entry.getKey();
|
||||
List<Map<String, Object>> sectionRows = entry.getValue();
|
||||
|
||||
if ("SESSION".equals(sectionName)) {
|
||||
renderSessionSection(md, sectionRows);
|
||||
} else if ("CORE METRICS".equals(sectionName)) {
|
||||
renderCoreMetricsSection(md, sectionRows);
|
||||
} else if ("ACTIVITY".equals(sectionName)) {
|
||||
renderActivitySection(md, sectionRows);
|
||||
} else if ("ERRORS".equals(sectionName)) {
|
||||
renderErrorsSection(md, sectionRows);
|
||||
} else if ("ALL METRICS".equals(sectionName)) {
|
||||
renderAllMetricsTree(md, sectionRows);
|
||||
} else {
|
||||
renderGenericSection(md, sectionName, sectionRows);
|
||||
}
|
||||
|
||||
md.append("\n");
|
||||
}
|
||||
|
||||
// Footer
|
||||
md.append("---\n\n");
|
||||
md.append(String.format("*Report generated in %dms*\n", result.executionTimeMs()));
|
||||
|
||||
return md.toString();
|
||||
}
|
||||
|
||||
private void renderSessionSection(StringBuilder md, List<Map<String, Object>> rows) {
|
||||
md.append("## 📊 Session Overview\n\n");
|
||||
|
||||
for (Map<String, Object> row : rows) {
|
||||
String metric = (String) row.get("metric");
|
||||
String value = (String) row.get("value");
|
||||
String details = (String) row.get("details");
|
||||
|
||||
md.append("**").append(metric).append(":** \n");
|
||||
md.append("`").append(value).append("`");
|
||||
if (details != null && !details.isEmpty()) {
|
||||
md.append(" — *").append(details).append("*");
|
||||
}
|
||||
md.append("\n\n");
|
||||
}
|
||||
}
|
||||
|
||||
private void renderCoreMetricsSection(StringBuilder md, List<Map<String, Object>> rows) {
|
||||
md.append("## 🎯 Core Metrics\n\n");
|
||||
|
||||
md.append("| Metric | Value | Details |\n");
|
||||
md.append("|--------|-------|----------|\n");
|
||||
|
||||
for (Map<String, Object> row : rows) {
|
||||
String metric = (String) row.get("metric");
|
||||
String value = (String) row.get("value");
|
||||
String details = (String) row.get("details");
|
||||
|
||||
md.append("| **").append(metric).append("** | ");
|
||||
md.append("`").append(value).append("` | ");
|
||||
md.append(details != null ? details : "");
|
||||
md.append(" |\n");
|
||||
}
|
||||
|
||||
md.append("\n");
|
||||
}
|
||||
|
||||
private void renderActivitySection(StringBuilder md, List<Map<String, Object>> rows) {
|
||||
md.append("## 📈 Activity Breakdown\n\n");
|
||||
|
||||
md.append("```\n");
|
||||
long total = rows.stream()
|
||||
.mapToLong(row -> {
|
||||
String value = (String) row.get("value");
|
||||
return (long) Double.parseDouble(value);
|
||||
})
|
||||
.sum();
|
||||
|
||||
for (Map<String, Object> row : rows) {
|
||||
String activity = (String) row.get("metric");
|
||||
String value = (String) row.get("value");
|
||||
long count = (long) Double.parseDouble(value);
|
||||
double percentage = (count * 100.0) / total;
|
||||
|
||||
md.append(String.format("%-15s %,12d ops (%5.1f%%)\n",
|
||||
activity + ":", count, percentage));
|
||||
}
|
||||
|
||||
md.append(String.format("%-15s %,12d ops\n", "TOTAL:", total));
|
||||
md.append("```\n\n");
|
||||
}
|
||||
|
||||
private void renderErrorsSection(StringBuilder md, List<Map<String, Object>> rows) {
|
||||
md.append("## ⚠️ Errors\n\n");
|
||||
|
||||
if (rows.isEmpty()) {
|
||||
md.append("*No errors recorded.*\n\n");
|
||||
return;
|
||||
}
|
||||
|
||||
md.append("| Error Type | Count |\n");
|
||||
md.append("|------------|-------|\n");
|
||||
|
||||
for (Map<String, Object> row : rows) {
|
||||
String metric = (String) row.get("metric");
|
||||
String value = (String) row.get("value");
|
||||
|
||||
md.append("| `").append(metric).append("` | ");
|
||||
md.append("**").append(value).append("** |\n");
|
||||
}
|
||||
|
||||
md.append("\n");
|
||||
}
|
||||
|
||||
private void renderAllMetricsTree(StringBuilder md, List<Map<String, Object>> rows) {
|
||||
md.append("## 📋 Metrics Inventory\n\n");
|
||||
|
||||
md.append("```\n");
|
||||
md.append("Metrics Database\n");
|
||||
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
Map<String, Object> row = rows.get(i);
|
||||
String name = (String) row.get("metric");
|
||||
String details = (String) row.get("details");
|
||||
|
||||
boolean isLast = (i == rows.size() - 1);
|
||||
String prefix = isLast ? "└── " : "├── ";
|
||||
|
||||
md.append(prefix).append(name).append("\n");
|
||||
|
||||
if (details != null && !details.isEmpty()) {
|
||||
String detailPrefix = isLast ? " " : "│ ";
|
||||
// Parse details and format as tree branches
|
||||
// Handle range=[min, max] specially to keep it on one line
|
||||
String[] parts = details.split(", (?![^\\[]*\\])");
|
||||
for (int j = 0; j < parts.length; j++) {
|
||||
boolean isLastDetail = (j == parts.length - 1);
|
||||
String branchPrefix = isLastDetail ? "└── " : "├── ";
|
||||
md.append(detailPrefix).append(branchPrefix).append(parts[j]).append("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
md.append("```\n\n");
|
||||
}
|
||||
|
||||
private void renderGenericSection(StringBuilder md, String sectionName, List<Map<String, Object>> rows) {
|
||||
md.append("## ").append(sectionName).append("\n\n");
|
||||
|
||||
md.append("| Metric | Value | Details |\n");
|
||||
md.append("|--------|-------|----------|\n");
|
||||
|
||||
for (Map<String, Object> row : rows) {
|
||||
String metric = (String) row.get("metric");
|
||||
String value = (String) row.get("value");
|
||||
String details = (String) row.get("details");
|
||||
|
||||
md.append("| ").append(metric).append(" | ");
|
||||
md.append(value != null ? value : "").append(" | ");
|
||||
md.append(details != null ? details : "");
|
||||
md.append(" |\n");
|
||||
}
|
||||
|
||||
md.append("\n");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "summary-markdown";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "md";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Formats query results as ASCII tables suitable for console output.
|
||||
*/
|
||||
public class TableFormatter implements ResultFormatter {
|
||||
|
||||
private static final int DEFAULT_MAX_COLUMN_WIDTH = 50;
|
||||
private static final String COLUMN_SEPARATOR = " | ";
|
||||
private static final String HEADER_SEPARATOR_CHAR = "-";
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
if (result.isEmpty()) {
|
||||
return "No results found.\n";
|
||||
}
|
||||
|
||||
List<String> columns = result.columns();
|
||||
List<Map<String, Object>> rows = result.rows();
|
||||
|
||||
// Calculate column widths
|
||||
Map<String, Integer> columnWidths = calculateColumnWidths(columns, rows);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Header
|
||||
sb.append(formatRow(columns, columnWidths, columns));
|
||||
sb.append("\n");
|
||||
|
||||
// Separator
|
||||
sb.append(formatSeparator(columns, columnWidths));
|
||||
sb.append("\n");
|
||||
|
||||
// Data rows
|
||||
for (Map<String, Object> row : rows) {
|
||||
List<String> values = new ArrayList<>();
|
||||
for (String column : columns) {
|
||||
Object value = row.get(column);
|
||||
values.add(value != null ? value.toString() : "null");
|
||||
}
|
||||
sb.append(formatRow(values, columnWidths, columns));
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
// Footer with row count
|
||||
sb.append("\n");
|
||||
sb.append(result.rowCount()).append(" row");
|
||||
if (result.rowCount() != 1) {
|
||||
sb.append("s");
|
||||
}
|
||||
sb.append(" (").append(result.executionTimeMs()).append("ms)\n");
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "table";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "txt";
|
||||
}
|
||||
|
||||
private Map<String, Integer> calculateColumnWidths(List<String> columns, List<Map<String, Object>> rows) {
|
||||
Map<String, Integer> widths = new java.util.LinkedHashMap<>();
|
||||
|
||||
// Initialize with header widths
|
||||
for (String column : columns) {
|
||||
widths.put(column, column.length());
|
||||
}
|
||||
|
||||
// Check data widths
|
||||
for (Map<String, Object> row : rows) {
|
||||
for (String column : columns) {
|
||||
Object value = row.get(column);
|
||||
String strValue = value != null ? value.toString() : "null";
|
||||
int currentWidth = widths.get(column);
|
||||
int valueWidth = Math.min(strValue.length(), DEFAULT_MAX_COLUMN_WIDTH);
|
||||
widths.put(column, Math.max(currentWidth, valueWidth));
|
||||
}
|
||||
}
|
||||
|
||||
return widths;
|
||||
}
|
||||
|
||||
private String formatRow(List<String> values, Map<String, Integer> columnWidths, List<String> columns) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String column = columns.get(i);
|
||||
String value = values.get(i);
|
||||
int width = columnWidths.get(column);
|
||||
|
||||
// Truncate if too long
|
||||
if (value.length() > DEFAULT_MAX_COLUMN_WIDTH) {
|
||||
value = value.substring(0, DEFAULT_MAX_COLUMN_WIDTH - 3) + "...";
|
||||
}
|
||||
|
||||
// Pad to width
|
||||
sb.append(padRight(value, width));
|
||||
|
||||
if (i < columns.size() - 1) {
|
||||
sb.append(COLUMN_SEPARATOR);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String formatSeparator(List<String> columns, Map<String, Integer> columnWidths) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String column = columns.get(i);
|
||||
int width = columnWidths.get(column);
|
||||
sb.append(HEADER_SEPARATOR_CHAR.repeat(width));
|
||||
|
||||
if (i < columns.size() - 1) {
|
||||
sb.append("-+-");
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String padRight(String str, int width) {
|
||||
if (str.length() >= width) {
|
||||
return str;
|
||||
}
|
||||
return str + " ".repeat(width - str.length());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.format;
|
||||
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Formats query results as TSV (Tab-Separated Values) for data processing tools.
|
||||
* Compatible with awk, cut, paste, and other Unix text processing utilities.
|
||||
*/
|
||||
public class TsvFormatter implements ResultFormatter {
|
||||
|
||||
private final boolean includeHeaders;
|
||||
|
||||
public TsvFormatter() {
|
||||
this(true);
|
||||
}
|
||||
|
||||
public TsvFormatter(boolean includeHeaders) {
|
||||
this.includeHeaders = includeHeaders;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String format(QueryResult result) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Header row
|
||||
if (includeHeaders) {
|
||||
sb.append(String.join("\t", result.columns()));
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
// Data rows
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
List<String> values = new ArrayList<>();
|
||||
for (String column : result.columns()) {
|
||||
Object value = row.get(column);
|
||||
// TSV doesn't require quoting - just convert to string
|
||||
// Tabs and newlines in data should be replaced
|
||||
String strValue = value != null ? value.toString() : "";
|
||||
strValue = strValue.replace("\t", " ").replace("\n", " ").replace("\r", "");
|
||||
values.add(strValue);
|
||||
}
|
||||
sb.append(String.join("\t", values));
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "tsv";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFileExtension() {
|
||||
return "tsv";
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.query;
|
||||
|
||||
/**
|
||||
* Exception thrown when query parameters are invalid.
|
||||
* Should provide clear, actionable error messages to help users correct their queries.
|
||||
*/
|
||||
public class InvalidQueryException extends Exception {
|
||||
|
||||
public InvalidQueryException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public InvalidQueryException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.query;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Interface for all MetricsQL query commands.
|
||||
* Each command translates high-level query parameters into SQL and executes against
|
||||
* the NoSQLBench metrics database.
|
||||
*/
|
||||
public interface MetricsQueryCommand {
|
||||
|
||||
/**
|
||||
* Get the command name (e.g., "rate", "instant", "agg").
|
||||
*/
|
||||
String getName();
|
||||
|
||||
/**
|
||||
* Get a human-readable description of what this command does.
|
||||
*/
|
||||
String getDescription();
|
||||
|
||||
/**
|
||||
* Execute the query command.
|
||||
*
|
||||
* @param conn Database connection
|
||||
* @param params Query parameters (metric name, time windows, labels, etc.)
|
||||
* @return Query result with columns and rows
|
||||
* @throws SQLException If query execution fails
|
||||
* @throws InvalidQueryException If parameters are invalid
|
||||
*/
|
||||
QueryResult execute(Connection conn, Map<String, Object> params) throws SQLException, InvalidQueryException;
|
||||
|
||||
/**
|
||||
* Validate query parameters before execution.
|
||||
* Should throw InvalidQueryException with clear error messages if parameters are invalid.
|
||||
*
|
||||
* @param params Query parameters to validate
|
||||
* @throws InvalidQueryException If parameters are invalid
|
||||
*/
|
||||
void validate(Map<String, Object> params) throws InvalidQueryException;
|
||||
|
||||
/**
|
||||
* Get usage examples for this command.
|
||||
* Used for help text and documentation.
|
||||
*/
|
||||
default String getUsageExamples() {
|
||||
return "No examples available.";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.query;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Represents the result of a metrics query.
|
||||
*
|
||||
* @param columns List of column names in order
|
||||
* @param rows List of row data as maps (column name -> value)
|
||||
* @param executedSQL The SQL query that was executed (for debugging/explain)
|
||||
* @param executionTimeMs Time taken to execute the query in milliseconds
|
||||
*/
|
||||
public record QueryResult(
|
||||
List<String> columns,
|
||||
List<Map<String, Object>> rows,
|
||||
String executedSQL,
|
||||
long executionTimeMs
|
||||
) {
|
||||
/**
|
||||
* Get the number of rows returned.
|
||||
*/
|
||||
public int rowCount() {
|
||||
return rows != null ? rows.size() : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the result is empty.
|
||||
*/
|
||||
public boolean isEmpty() {
|
||||
return rowCount() == 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.query;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Parses time window specifications like "5m", "1h", "30s" into milliseconds.
|
||||
* Supports PromQL/MetricsQL time duration format.
|
||||
*/
|
||||
public class TimeWindowParser {
|
||||
|
||||
// Pattern: number followed by time unit (s, m, h, d, w, y)
|
||||
private static final Pattern TIME_WINDOW_PATTERN = Pattern.compile("^(\\d+(?:\\.\\d+)?)([smhdwy])$");
|
||||
|
||||
/**
|
||||
* Parse a time window string into milliseconds.
|
||||
*
|
||||
* Supported formats:
|
||||
* - "5s" = 5 seconds = 5000ms
|
||||
* - "10m" = 10 minutes = 600000ms
|
||||
* - "2h" = 2 hours = 7200000ms
|
||||
* - "1d" = 1 day = 86400000ms
|
||||
* - "1w" = 1 week = 604800000ms
|
||||
* - "1y" = 1 year (365 days) = 31536000000ms
|
||||
*
|
||||
* @param window Time window string (e.g., "5m", "1h")
|
||||
* @return Time window in milliseconds
|
||||
* @throws InvalidQueryException If the format is invalid
|
||||
*/
|
||||
public static long parseToMillis(String window) throws InvalidQueryException {
|
||||
if (window == null || window.trim().isEmpty()) {
|
||||
throw new InvalidQueryException("Time window cannot be null or empty");
|
||||
}
|
||||
|
||||
String normalized = window.trim().toLowerCase();
|
||||
Matcher matcher = TIME_WINDOW_PATTERN.matcher(normalized);
|
||||
|
||||
if (!matcher.matches()) {
|
||||
throw new InvalidQueryException(
|
||||
"Invalid time window format: '" + window + "'\n" +
|
||||
"Expected format: <number><unit> where unit is one of: s, m, h, d, w, y\n" +
|
||||
"Examples: 5s, 10m, 2h, 1d"
|
||||
);
|
||||
}
|
||||
|
||||
double value;
|
||||
try {
|
||||
value = Double.parseDouble(matcher.group(1));
|
||||
} catch (NumberFormatException e) {
|
||||
throw new InvalidQueryException("Invalid numeric value in time window: " + window, e);
|
||||
}
|
||||
|
||||
if (value <= 0) {
|
||||
throw new InvalidQueryException("Time window must be positive, got: " + value);
|
||||
}
|
||||
|
||||
String unit = matcher.group(2);
|
||||
long millisPerUnit = getMillisPerUnit(unit);
|
||||
|
||||
// Check for overflow
|
||||
double resultMs = value * millisPerUnit;
|
||||
if (resultMs > Long.MAX_VALUE) {
|
||||
throw new InvalidQueryException("Time window too large: " + window);
|
||||
}
|
||||
|
||||
return (long) resultMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a time window string, throwing RuntimeException on error.
|
||||
* Useful for constants where you know the format is valid.
|
||||
*
|
||||
* @param window Time window string
|
||||
* @return Time window in milliseconds
|
||||
* @throws RuntimeException If parsing fails
|
||||
*/
|
||||
public static long parseToMillisUnchecked(String window) {
|
||||
try {
|
||||
return parseToMillis(window);
|
||||
} catch (InvalidQueryException e) {
|
||||
throw new RuntimeException("Failed to parse time window: " + window, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format milliseconds back to a human-readable time window string.
|
||||
* Chooses the most appropriate unit automatically.
|
||||
*
|
||||
* @param millis Time in milliseconds
|
||||
* @return Human-readable string (e.g., "5m", "2h")
|
||||
*/
|
||||
public static String formatMillis(long millis) {
|
||||
if (millis == 0) {
|
||||
return "0s";
|
||||
}
|
||||
|
||||
// Try to find the largest unit that divides evenly
|
||||
if (millis % MILLIS_PER_YEAR == 0) {
|
||||
return (millis / MILLIS_PER_YEAR) + "y";
|
||||
}
|
||||
if (millis % MILLIS_PER_WEEK == 0) {
|
||||
return (millis / MILLIS_PER_WEEK) + "w";
|
||||
}
|
||||
if (millis % MILLIS_PER_DAY == 0) {
|
||||
return (millis / MILLIS_PER_DAY) + "d";
|
||||
}
|
||||
if (millis % MILLIS_PER_HOUR == 0) {
|
||||
return (millis / MILLIS_PER_HOUR) + "h";
|
||||
}
|
||||
if (millis % MILLIS_PER_MINUTE == 0) {
|
||||
return (millis / MILLIS_PER_MINUTE) + "m";
|
||||
}
|
||||
if (millis % MILLIS_PER_SECOND == 0) {
|
||||
return (millis / MILLIS_PER_SECOND) + "s";
|
||||
}
|
||||
|
||||
// Fallback to milliseconds
|
||||
return millis + "ms";
|
||||
}
|
||||
|
||||
// Time unit constants
|
||||
private static final long MILLIS_PER_SECOND = 1000L;
|
||||
private static final long MILLIS_PER_MINUTE = 60L * MILLIS_PER_SECOND;
|
||||
private static final long MILLIS_PER_HOUR = 60L * MILLIS_PER_MINUTE;
|
||||
private static final long MILLIS_PER_DAY = 24L * MILLIS_PER_HOUR;
|
||||
private static final long MILLIS_PER_WEEK = 7L * MILLIS_PER_DAY;
|
||||
private static final long MILLIS_PER_YEAR = 365L * MILLIS_PER_DAY; // No leap years in PromQL
|
||||
|
||||
private static long getMillisPerUnit(String unit) throws InvalidQueryException {
|
||||
return switch (unit) {
|
||||
case "s" -> MILLIS_PER_SECOND;
|
||||
case "m" -> MILLIS_PER_MINUTE;
|
||||
case "h" -> MILLIS_PER_HOUR;
|
||||
case "d" -> MILLIS_PER_DAY;
|
||||
case "w" -> MILLIS_PER_WEEK;
|
||||
case "y" -> MILLIS_PER_YEAR;
|
||||
default -> throw new InvalidQueryException("Unknown time unit: " + unit);
|
||||
};
|
||||
}
|
||||
|
||||
private TimeWindowParser() {
|
||||
// Utility class - prevent instantiation
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.schema;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
/**
|
||||
* Utility for computing label set hashes compatible with SqliteSnapshotReporter.
|
||||
* This logic is intentionally duplicated from SqliteSnapshotReporter to maintain
|
||||
* module independence (nb-api doesn't depend on nb-mql-api).
|
||||
*/
|
||||
public class LabelSetResolver {
|
||||
|
||||
/**
|
||||
* Compute a deterministic hash string for a label set.
|
||||
* Format: {key1=value1,key2=value2,...} with keys sorted alphabetically.
|
||||
*
|
||||
* @param labels Map of label key-value pairs
|
||||
* @return Hash string representation
|
||||
*/
|
||||
public static String computeHash(Map<String, String> labels) {
|
||||
if (labels == null || labels.isEmpty()) {
|
||||
return "{}";
|
||||
}
|
||||
StringJoiner joiner = new StringJoiner(",", "{", "}");
|
||||
labels.entrySet().stream()
|
||||
.sorted(Map.Entry.comparingByKey())
|
||||
.forEach(entry -> joiner.add(entry.getKey() + "=" + entry.getValue()));
|
||||
return joiner.toString();
|
||||
}
|
||||
|
||||
private LabelSetResolver() {
|
||||
// Utility class - prevent instantiation
|
||||
}
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.schema;
|
||||
|
||||
import org.sqlite.SQLiteConfig;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
|
||||
/**
|
||||
* Manages read-only connections to NoSQLBench SQLite metrics databases.
|
||||
* Supports querying both live sessions (via logs/metrics.db symlink) and
|
||||
* historical session files.
|
||||
*/
|
||||
public class MetricsDatabaseReader {
|
||||
|
||||
private static final Path DEFAULT_DB_PATH = Path.of("logs/metrics.db");
|
||||
private static final int DEFAULT_BUSY_TIMEOUT_MS = 5000;
|
||||
|
||||
static {
|
||||
// Explicitly load SQLite JDBC driver for standalone JAR usage
|
||||
try {
|
||||
Class.forName("org.sqlite.JDBC");
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new RuntimeException("SQLite JDBC driver not found", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to the default metrics database (logs/metrics.db).
|
||||
* This typically points to the current session via symlink.
|
||||
*
|
||||
* @return Read-only database connection
|
||||
* @throws SQLException If connection fails or database doesn't exist
|
||||
*/
|
||||
public static Connection connect() throws SQLException {
|
||||
return connect(DEFAULT_DB_PATH);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to a specific metrics database file.
|
||||
*
|
||||
* @param dbPath Path to the database file (can be a symlink)
|
||||
* @return Read-only database connection
|
||||
* @throws SQLException If connection fails or database doesn't exist
|
||||
*/
|
||||
public static Connection connect(Path dbPath) throws SQLException {
|
||||
return connect(dbPath, DEFAULT_BUSY_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to a specific metrics database with custom busy timeout.
|
||||
*
|
||||
* @param dbPath Path to the database file
|
||||
* @param busyTimeoutMs Timeout in milliseconds for handling database locks
|
||||
* @return Read-only database connection
|
||||
* @throws SQLException If connection fails or database doesn't exist
|
||||
*/
|
||||
public static Connection connect(Path dbPath, int busyTimeoutMs) throws SQLException {
|
||||
Path resolvedPath = (dbPath != null) ? dbPath : DEFAULT_DB_PATH;
|
||||
|
||||
// Check if database file exists
|
||||
if (!Files.exists(resolvedPath)) {
|
||||
throw new SQLException(
|
||||
"Database not found at: " + resolvedPath.toAbsolutePath() + "\n" +
|
||||
"Hint: Ensure a NoSQLBench session is running or specify a session file with --db <path>"
|
||||
);
|
||||
}
|
||||
|
||||
// Build JDBC URL
|
||||
String jdbcUrl = "jdbc:sqlite:" + resolvedPath.toAbsolutePath();
|
||||
|
||||
// Configure SQLite for read-only access
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setReadOnly(true);
|
||||
config.setBusyTimeout(busyTimeoutMs);
|
||||
|
||||
// Create connection with config
|
||||
return DriverManager.getConnection(jdbcUrl, config.toProperties());
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to a database specified by JDBC URL string.
|
||||
* Useful for testing with in-memory databases.
|
||||
*
|
||||
* @param jdbcUrl JDBC URL (e.g., "jdbc:sqlite::memory:")
|
||||
* @return Read-only database connection
|
||||
* @throws SQLException If connection fails
|
||||
*/
|
||||
public static Connection connectByUrl(String jdbcUrl) throws SQLException {
|
||||
SQLiteConfig config = new SQLiteConfig();
|
||||
config.setReadOnly(true);
|
||||
config.setBusyTimeout(DEFAULT_BUSY_TIMEOUT_MS);
|
||||
|
||||
return DriverManager.getConnection(jdbcUrl, config.toProperties());
|
||||
}
|
||||
|
||||
private MetricsDatabaseReader() {
|
||||
// Utility class - prevent instantiation
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.schema;
|
||||
|
||||
/**
|
||||
* Schema constants for NoSQLBench SQLite metrics database.
|
||||
* This schema is defined and created by SqliteSnapshotReporter in nb-api.
|
||||
* These constants provide a read-only view for query operations.
|
||||
*/
|
||||
public class MetricsSchema {
|
||||
|
||||
// Table names
|
||||
public static final String TABLE_METRIC_FAMILY = "metric_family";
|
||||
public static final String TABLE_SAMPLE_NAME = "sample_name";
|
||||
public static final String TABLE_SAMPLE_VALUE = "sample_value";
|
||||
public static final String TABLE_SAMPLE_QUANTILE = "sample_quantile";
|
||||
public static final String TABLE_SAMPLE_RATE = "sample_rate";
|
||||
public static final String TABLE_SAMPLE_HISTOGRAM = "sample_histogram";
|
||||
public static final String TABLE_LABEL_KEY = "label_key";
|
||||
public static final String TABLE_LABEL_VALUE = "label_value";
|
||||
public static final String TABLE_LABEL_SET = "label_set";
|
||||
public static final String TABLE_LABEL_SET_MEMBERSHIP = "label_set_membership";
|
||||
|
||||
// metric_family columns
|
||||
public static final String COL_MF_ID = "id";
|
||||
public static final String COL_MF_NAME = "name";
|
||||
public static final String COL_MF_HELP = "help";
|
||||
public static final String COL_MF_UNIT = "unit";
|
||||
public static final String COL_MF_TYPE = "type";
|
||||
|
||||
// sample_name columns
|
||||
public static final String COL_SN_ID = "id";
|
||||
public static final String COL_SN_METRIC_FAMILY_ID = "metric_family_id";
|
||||
public static final String COL_SN_SAMPLE = "sample";
|
||||
|
||||
// sample_value columns
|
||||
public static final String COL_SV_ID = "id";
|
||||
public static final String COL_SV_SAMPLE_NAME_ID = "sample_name_id";
|
||||
public static final String COL_SV_LABEL_SET_ID = "label_set_id";
|
||||
public static final String COL_SV_TIMESTAMP_MS = "timestamp_ms";
|
||||
public static final String COL_SV_VALUE = "value";
|
||||
public static final String COL_SV_COUNT = "count";
|
||||
public static final String COL_SV_SUM = "sum";
|
||||
public static final String COL_SV_MIN = "min";
|
||||
public static final String COL_SV_MAX = "max";
|
||||
public static final String COL_SV_MEAN = "mean";
|
||||
public static final String COL_SV_STDDEV = "stddev";
|
||||
|
||||
// sample_quantile columns
|
||||
public static final String COL_SQ_SAMPLE_VALUE_ID = "sample_value_id";
|
||||
public static final String COL_SQ_QUANTILE = "quantile";
|
||||
public static final String COL_SQ_QUANTILE_VALUE = "quantile_value";
|
||||
|
||||
// sample_rate columns
|
||||
public static final String COL_SR_SAMPLE_VALUE_ID = "sample_value_id";
|
||||
public static final String COL_SR_RATE_TYPE = "rate_type";
|
||||
public static final String COL_SR_RATE_VALUE = "rate_value";
|
||||
|
||||
// sample_histogram columns
|
||||
public static final String COL_SH_SAMPLE_VALUE_ID = "sample_value_id";
|
||||
public static final String COL_SH_START_SECONDS = "start_seconds";
|
||||
public static final String COL_SH_INTERVAL_SECONDS = "interval_seconds";
|
||||
public static final String COL_SH_MAX_VALUE = "max_value";
|
||||
public static final String COL_SH_HISTOGRAM_BASE64 = "histogram_base64";
|
||||
|
||||
// label_key columns
|
||||
public static final String COL_LK_ID = "id";
|
||||
public static final String COL_LK_NAME = "name";
|
||||
|
||||
// label_value columns
|
||||
public static final String COL_LV_ID = "id";
|
||||
public static final String COL_LV_VALUE = "value";
|
||||
|
||||
// label_set columns
|
||||
public static final String COL_LS_ID = "id";
|
||||
public static final String COL_LS_HASH = "hash";
|
||||
|
||||
// label_set_membership columns
|
||||
public static final String COL_LSM_LABEL_SET_ID = "label_set_id";
|
||||
public static final String COL_LSM_LABEL_KEY_ID = "label_key_id";
|
||||
public static final String COL_LSM_LABEL_VALUE_ID = "label_value_id";
|
||||
|
||||
// Common SQL fragments for building queries
|
||||
|
||||
/**
|
||||
* SQL fragment to join sample_value with sample_name.
|
||||
* Usage: FROM sample_value sv JOIN sample_name sn ON ...
|
||||
*/
|
||||
public static String joinSampleName() {
|
||||
return String.format("%s.%s = %s.%s",
|
||||
"sn", COL_SN_ID,
|
||||
"sv", COL_SV_SAMPLE_NAME_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL fragment to join sample_value with label_set.
|
||||
* Usage: FROM sample_value sv JOIN label_set ls ON ...
|
||||
*/
|
||||
public static String joinLabelSet() {
|
||||
return String.format("%s.%s = %s.%s",
|
||||
"ls", COL_LS_ID,
|
||||
"sv", COL_SV_LABEL_SET_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL fragment to join label_set with label_set_membership.
|
||||
* Usage: JOIN label_set_membership lsm ON ...
|
||||
*/
|
||||
public static String joinLabelSetMembership() {
|
||||
return String.format("%s.%s = %s.%s",
|
||||
"lsm", COL_LSM_LABEL_SET_ID,
|
||||
"ls", COL_LS_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL fragment to join label_set_membership with label_key.
|
||||
* Usage: JOIN label_key lk ON ...
|
||||
*/
|
||||
public static String joinLabelKey() {
|
||||
return String.format("%s.%s = %s.%s",
|
||||
"lk", COL_LK_ID,
|
||||
"lsm", COL_LSM_LABEL_KEY_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* SQL fragment to join label_set_membership with label_value.
|
||||
* Usage: JOIN label_value lv ON ...
|
||||
*/
|
||||
public static String joinLabelValue() {
|
||||
return String.format("%s.%s = %s.%s",
|
||||
"lv", COL_LV_ID,
|
||||
"lsm", COL_LSM_LABEL_VALUE_ID);
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete SQL fragment for joining all label-related tables.
|
||||
* Produces: JOIN label_set ls ON ... LEFT JOIN label_set_membership lsm ON ... etc.
|
||||
*/
|
||||
public static String joinAllLabels() {
|
||||
return String.format(
|
||||
"JOIN %s ls ON %s " +
|
||||
"LEFT JOIN %s lsm ON %s " +
|
||||
"LEFT JOIN %s lk ON %s " +
|
||||
"LEFT JOIN %s lv ON %s",
|
||||
TABLE_LABEL_SET, joinLabelSet(),
|
||||
TABLE_LABEL_SET_MEMBERSHIP, joinLabelSetMembership(),
|
||||
TABLE_LABEL_KEY, joinLabelKey(),
|
||||
TABLE_LABEL_VALUE, joinLabelValue()
|
||||
);
|
||||
}
|
||||
|
||||
private MetricsSchema() {
|
||||
// Utility class - prevent instantiation
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
io.nosqlbench.nb.mql.MQLApp
|
||||
@@ -0,0 +1,124 @@
|
||||
# MetricsQL Query Examples
|
||||
|
||||
This directory contains executable examples demonstrating all query commands in the MetricsQL query engine. Each example is validated by automated tests to ensure the documentation stays accurate.
|
||||
|
||||
## Example Files by Theme
|
||||
|
||||
### Basic Queries
|
||||
- **[INSTANT_QUERIES.md](INSTANT_QUERIES.md)** - Get current/latest values
|
||||
- Basic instant queries
|
||||
- Label filtering
|
||||
- Error metrics
|
||||
- Multi-dimensional filtering
|
||||
|
||||
- **[RANGE_QUERIES.md](RANGE_QUERIES.md)** - Time-series data retrieval
|
||||
- Time-series over windows
|
||||
- Historical data analysis
|
||||
- Error tracking over time
|
||||
|
||||
### Analysis Queries (Coming Soon)
|
||||
- **RATE_QUERIES.md** - Calculate per-second rates
|
||||
- Counter rate calculations
|
||||
- Change detection
|
||||
- Throughput analysis
|
||||
|
||||
- **AGGREGATION_QUERIES.md** - Sum, average, min, max operations
|
||||
- Grouping by labels
|
||||
- Cross-dimensional aggregations
|
||||
- Statistical summaries
|
||||
|
||||
- **QUANTILE_QUERIES.md** - Percentile analysis
|
||||
- Latency percentiles (p50, p95, p99)
|
||||
- Timer statistics
|
||||
- Performance analysis
|
||||
|
||||
### Advanced Queries (Future)
|
||||
- **RATIO_QUERIES.md** - Calculate error rates and ratios
|
||||
- **TOPK_QUERIES.md** - Find top N values
|
||||
- **BINARY_OPERATIONS.md** - Arithmetic between metrics
|
||||
|
||||
## Test Database: examples.db
|
||||
|
||||
All examples use `examples.db`, which contains metrics from a simplified e-commerce API:
|
||||
|
||||
**Metrics:**
|
||||
- `api_requests_total` - Request counters by method, endpoint, and status
|
||||
- `api_latency_ms` - API latency timers with percentiles
|
||||
|
||||
**Timeline:**
|
||||
- 5 snapshots spanning 10 minutes (2025-10-23 10:00:00Z to 10:10:00Z)
|
||||
- Simulates normal traffic → increased load → high load → recovery
|
||||
|
||||
**Label Dimensions:**
|
||||
- `method`: {GET, POST}
|
||||
- `endpoint`: {/users, /products}
|
||||
- `status`: {200, 404, 500}
|
||||
|
||||
See [TEST_DATABASES.md](../TEST_DATABASES.md) for detailed schema information.
|
||||
|
||||
## How Examples Work
|
||||
|
||||
Each example follows this format:
|
||||
|
||||
```markdown
|
||||
## Example N: Description
|
||||
|
||||
**Command:**
|
||||
```
|
||||
command --param value --other param
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
[exact formatted table output]
|
||||
```
|
||||
|
||||
**What it shows:** Explanation of the use case
|
||||
```
|
||||
|
||||
The `ExampleDocumentationTest` class automatically:
|
||||
1. Parses all example markdown files
|
||||
2. Extracts commands and expected results
|
||||
3. Executes each command against examples.db
|
||||
4. Validates the actual output matches expectations
|
||||
|
||||
**This means documentation is always accurate and tested!**
|
||||
|
||||
## Running Examples Manually
|
||||
|
||||
Once the CLI is implemented, you can run these examples:
|
||||
|
||||
```bash
|
||||
# Point to the examples database
|
||||
export EXAMPLES_DB=src/test/resources/testdata/examples.db
|
||||
|
||||
# Run any example command
|
||||
nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total
|
||||
|
||||
# With different output formats
|
||||
nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total --format json
|
||||
nb-metricsql instant --db $EXAMPLES_DB --metric api_requests_total --format csv
|
||||
```
|
||||
|
||||
## Regenerating examples.db
|
||||
|
||||
If you modify the schema or need to update the example data:
|
||||
|
||||
```bash
|
||||
mvn exec:java -pl nb-apis/nb-mql-api \
|
||||
-Dexec.mainClass="io.nosqlbench.nb.mql.testdata.ExampleDataGenerator" \
|
||||
-Dexec.classpathScope=test
|
||||
```
|
||||
|
||||
## Contributing New Examples
|
||||
|
||||
When adding a new query command:
|
||||
|
||||
1. Add examples to the appropriate themed markdown file
|
||||
2. Follow the established format with command, expected output, and explanation
|
||||
3. Run the tests to verify: `mvn test -Dtest=ExampleDocumentationTest`
|
||||
4. The tests will automatically validate your examples
|
||||
|
||||
---
|
||||
|
||||
*These examples are automatically tested and validated with every build.*
|
||||
@@ -0,0 +1,130 @@
|
||||
# Instant Queries
|
||||
|
||||
Get the most recent value for metrics. Instant queries return the latest snapshot for each unique label set.
|
||||
|
||||
**Database:** `examples.db`
|
||||
|
||||
---
|
||||
|
||||
## Example 1: Get All Latest Values
|
||||
|
||||
**Command:**
|
||||
```
|
||||
instant --metric api_requests_total
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
timestamp | value | labels
|
||||
----------------------------------------------------------------------------------
|
||||
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
|
||||
2025-10-23 10:10:00.0 | 8500.0 | endpoint=/products, method=GET, status=200
|
||||
2025-10-23 10:10:00.0 | 1500.0 | endpoint=/users, method=POST, status=200
|
||||
2025-10-23 10:10:00.0 | 75.0 | endpoint=/users, method=GET, status=404
|
||||
2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500
|
||||
|
||||
5 rows (Xms)
|
||||
```
|
||||
|
||||
**What it shows:** Retrieves the latest value for all label combinations of a metric. Useful for dashboards showing current state.
|
||||
|
||||
**Use cases:**
|
||||
- Current request counts by endpoint
|
||||
- Latest error counts
|
||||
- Current gauge values
|
||||
|
||||
---
|
||||
|
||||
## Example 2: Filter by Specific Labels
|
||||
|
||||
**Command:**
|
||||
```
|
||||
instant --metric api_requests_total --labels method=GET,endpoint=/users,status=200
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
timestamp | value | labels
|
||||
----------------------------------------------------------------------------------
|
||||
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
|
||||
|
||||
1 row (Xms)
|
||||
```
|
||||
|
||||
**What it shows:** Filtering to a specific label combination returns a single time series.
|
||||
|
||||
**Use cases:**
|
||||
- Monitor specific endpoint traffic
|
||||
- Track particular operation types
|
||||
- Isolate specific service behavior
|
||||
|
||||
---
|
||||
|
||||
## Example 3: Query Only Errors
|
||||
|
||||
**Command:**
|
||||
```
|
||||
instant --metric api_requests_total --labels status=500
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
timestamp | value | labels
|
||||
--------------------------------------------------------------------------------
|
||||
2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500
|
||||
|
||||
1 row (Xms)
|
||||
```
|
||||
|
||||
**What it shows:** Filter by a single label dimension (status code) to see current error count.
|
||||
|
||||
**Use cases:**
|
||||
- Current error count for alerting
|
||||
- Quick health check
|
||||
- SLA monitoring
|
||||
|
||||
---
|
||||
|
||||
## Example 4: Filter Across Multiple Series
|
||||
|
||||
**Command:**
|
||||
```
|
||||
instant --metric api_requests_total --labels status=200
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
timestamp | value | labels
|
||||
----------------------------------------------------------------------------------
|
||||
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
|
||||
2025-10-23 10:10:00.0 | 8500.0 | endpoint=/products, method=GET, status=200
|
||||
2025-10-23 10:10:00.0 | 1500.0 | endpoint=/users, method=POST, status=200
|
||||
|
||||
3 rows (Xms)
|
||||
```
|
||||
|
||||
**What it shows:** Filtering by a shared label value returns all matching label sets. This groups all successful requests while preserving endpoint/method distinctions.
|
||||
|
||||
**Use cases:**
|
||||
- All successful requests across endpoints
|
||||
- All error types (status >= 400)
|
||||
- Traffic patterns by method type
|
||||
|
||||
---
|
||||
|
||||
## Key Concepts
|
||||
|
||||
**Label Filtering:**
|
||||
- Specify labels as comma-separated key=value pairs
|
||||
- All specified labels must match (AND logic)
|
||||
- Unspecified labels can have any value
|
||||
|
||||
**Output:**
|
||||
- `timestamp`: When the snapshot was captured
|
||||
- `value`: The metric value at that time
|
||||
- `labels`: All labels for this time series (comma-separated)
|
||||
|
||||
**Performance:**
|
||||
- Instant queries are fast (typically <5ms)
|
||||
- Queries only the latest snapshot
|
||||
- Efficient for real-time dashboards
|
||||
@@ -0,0 +1,93 @@
|
||||
# Range Queries
|
||||
|
||||
Retrieve time-series data over a time window. Range queries return all data points within a specified time range.
|
||||
|
||||
**Database:** `examples.db`
|
||||
|
||||
---
|
||||
|
||||
## Example 5: Time Series - Full History
|
||||
|
||||
**Command:**
|
||||
```
|
||||
range --metric api_requests_total --window 1h --labels method=GET,endpoint=/users,status=200
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
timestamp | value | labels
|
||||
----------------------------------------------------------------------------------
|
||||
2025-10-23 10:00:00.0 | 0.0 | endpoint=/users, method=GET, status=200
|
||||
2025-10-23 10:01:00.0 | 1000.0 | endpoint=/users, method=GET, status=200
|
||||
2025-10-23 10:02:00.0 | 2200.0 | endpoint=/users, method=GET, status=200
|
||||
2025-10-23 10:05:00.0 | 5500.0 | endpoint=/users, method=GET, status=200
|
||||
2025-10-23 10:10:00.0 | 11000.0 | endpoint=/users, method=GET, status=200
|
||||
|
||||
5 rows (Xms)
|
||||
```
|
||||
|
||||
**What it shows:** Complete time-series progression showing how a counter grew from 0 to 11,000 requests over 10 minutes.
|
||||
|
||||
**Use cases:**
|
||||
- Visualize traffic patterns over time
|
||||
- Analyze growth trends
|
||||
- Export to graphing tools
|
||||
- Historical analysis
|
||||
|
||||
---
|
||||
|
||||
## Example 6: Track Errors Over Time
|
||||
|
||||
**Command:**
|
||||
```
|
||||
range --metric api_requests_total --window 1h --labels status=500
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
timestamp | value | labels
|
||||
--------------------------------------------------------------------------------
|
||||
2025-10-23 10:00:00.0 | 0.0 | endpoint=/users, method=GET, status=500
|
||||
2025-10-23 10:01:00.0 | 2.0 | endpoint=/users, method=GET, status=500
|
||||
2025-10-23 10:02:00.0 | 4.0 | endpoint=/users, method=GET, status=500
|
||||
2025-10-23 10:05:00.0 | 8.0 | endpoint=/users, method=GET, status=500
|
||||
2025-10-23 10:10:00.0 | 15.0 | endpoint=/users, method=GET, status=500
|
||||
|
||||
5 rows (Xms)
|
||||
```
|
||||
|
||||
**What it shows:** Track how server errors accumulated during the session. Notice the accelerated error rate during the high-load period (10:02 to 10:05).
|
||||
|
||||
**Use cases:**
|
||||
- Incident investigation
|
||||
- Error rate trending
|
||||
- Correlate errors with load spikes
|
||||
- SLA violation detection
|
||||
|
||||
---
|
||||
|
||||
## Key Concepts
|
||||
|
||||
**Time Windows:**
|
||||
- `--window 5m` - Last 5 minutes from latest snapshot
|
||||
- `--window 1h` - Last hour
|
||||
- `--last 10m` - Alternative syntax for --window
|
||||
|
||||
**Alternative: Explicit Time Range (Future):**
|
||||
```
|
||||
range --metric api_requests_total --start 1729681200000 --end 1729684800000
|
||||
```
|
||||
|
||||
**Output Ordering:**
|
||||
- Results are ordered by timestamp (oldest first)
|
||||
- Within same timestamp, ordered by labels alphabetically
|
||||
|
||||
**Performance:**
|
||||
- Range queries scan all data in the time window
|
||||
- Use label filters to reduce result set
|
||||
- Typical query time: <20ms for 1 hour of data
|
||||
|
||||
**Visualization:**
|
||||
- Export to CSV for Excel/Sheets: `--format csv`
|
||||
- Export to JSON for plotting libraries: `--format json`
|
||||
- Table format for quick console analysis
|
||||
@@ -0,0 +1,135 @@
|
||||
# MetricsQL Query Engine
|
||||
|
||||
MetricsQL is a query engine for analyzing NoSQLBench SQLite metrics databases. It provides a MetricsQL/PromQL-inspired interface for querying time-series metrics data.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Show session information
|
||||
nb5 mql session
|
||||
|
||||
# List available metrics
|
||||
nb5 mql summary
|
||||
|
||||
# Get current metric values
|
||||
nb5 mql instant --metric api_requests_total
|
||||
|
||||
# Query time-series data
|
||||
nb5 mql range --metric ops_total --window 5m
|
||||
|
||||
# Calculate request rate
|
||||
nb5 mql rate --metric requests_total --window 5m
|
||||
```
|
||||
|
||||
## Available Commands
|
||||
|
||||
### Discovery Commands
|
||||
- **session** - Display session timeline and snapshot information
|
||||
- **summary** - List all metrics with types and sample counts
|
||||
- **sql** - Execute raw SQLite queries for advanced analysis
|
||||
|
||||
### Query Commands
|
||||
- **instant** - Get the most recent value for a metric
|
||||
- **range** - Retrieve time-series data over a time window
|
||||
- **topk** - Find top N label sets by metric value
|
||||
|
||||
### Calculation Commands
|
||||
- **rate** - Calculate per-second rate of change for counters
|
||||
- **increase** - Calculate total increase in counter over time window
|
||||
- **quantile** - Extract percentile values from timer/summary metrics
|
||||
- **agg** - Perform aggregations (sum, avg, max, min, count) with grouping
|
||||
- **ratio** - Calculate ratio between two metrics (e.g., error rate)
|
||||
|
||||
## Output Formats
|
||||
|
||||
All commands support multiple output formats via `--format`:
|
||||
|
||||
- **table** (default) - ASCII tables for console
|
||||
- **json** - Pretty-printed JSON for programmatic use
|
||||
- **jsonl** - JSON Lines for streaming
|
||||
- **csv** - Comma-separated values for spreadsheets
|
||||
- **tsv** - Tab-separated values for Unix tools
|
||||
- **markdown** - GitHub-flavored markdown tables
|
||||
|
||||
## Examples
|
||||
|
||||
See detailed examples organized by topic:
|
||||
- [Examples Overview](mql-examples.md) - Navigation and all examples
|
||||
- [Instant Queries](mql-instant-queries.md) - Point-in-time queries
|
||||
- [Range Queries](mql-range-queries.md) - Time-series queries
|
||||
- More examples coming for rate, aggregation, and quantile queries
|
||||
|
||||
## Database Location
|
||||
|
||||
By default, commands query `logs/metrics.db` which is a symlink to the current session's metrics database.
|
||||
|
||||
To query a specific session:
|
||||
```bash
|
||||
nb5 mql instant --db logs/session_20251023_100000_metrics.db --metric ops_total
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Monitoring
|
||||
```bash
|
||||
# Current error count
|
||||
nb5 mql instant --metric api_requests_total --labels status=500
|
||||
|
||||
# Request rate last 5 minutes
|
||||
nb5 mql rate --metric api_requests_total --window 5m --labels status=200
|
||||
|
||||
# p95 latency
|
||||
nb5 mql quantile --metric api_latency --p 0.95
|
||||
```
|
||||
|
||||
### Analysis
|
||||
```bash
|
||||
# Total requests by endpoint
|
||||
nb5 mql agg --metric api_requests_total --func sum --by endpoint
|
||||
|
||||
# Top 10 busiest endpoints
|
||||
nb5 mql topk --metric api_requests_total --n 10
|
||||
|
||||
# Error rate
|
||||
nb5 mql ratio --numerator errors_total --denominator requests_total
|
||||
```
|
||||
|
||||
### Exporting Data
|
||||
```bash
|
||||
# CSV for Excel
|
||||
nb5 mql range --metric ops_total --window 1h --format csv > metrics.csv
|
||||
|
||||
# JSON for analysis
|
||||
nb5 mql summary --format json > metrics_inventory.json
|
||||
|
||||
# Streaming JSONL
|
||||
nb5 mql range --metric ops_total --window 1h --format jsonl | jq '.value'
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### SQL Queries
|
||||
```bash
|
||||
# Custom aggregation
|
||||
nb5 mql sql --query "
|
||||
SELECT sn.sample, AVG(sv.value) as avg_value
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
GROUP BY sn.sample
|
||||
"
|
||||
|
||||
# Schema inspection
|
||||
nb5 mql sql --query "PRAGMA table_info(sample_value)"
|
||||
```
|
||||
|
||||
### Debug Mode
|
||||
```bash
|
||||
# Show generated SQL
|
||||
nb5 mql instant --metric ops_total --explain
|
||||
```
|
||||
|
||||
## Learn More
|
||||
|
||||
Run `nb5 help mql` to see this documentation.
|
||||
|
||||
For detailed examples, see the [examples README](docs/mql/README.md).
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class AggregateCommandTest {
|
||||
|
||||
@Test
|
||||
void testSumNoGrouping() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"function", "sum"
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 1 row with total sum
|
||||
assertEquals(1, result.rowCount());
|
||||
assertEquals(1, result.columns().size());
|
||||
assertEquals("sum", result.columns().get(0));
|
||||
|
||||
System.out.println("\n=== Aggregate: Sum (No Grouping) ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSumByActivity() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"function", "sum",
|
||||
"by", List.of("activity")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 2 rows (read and write)
|
||||
assertEquals(2, result.rowCount(), "Should have 2 activities");
|
||||
assertEquals(2, result.columns().size());
|
||||
assertTrue(result.columns().contains("activity"));
|
||||
assertTrue(result.columns().contains("sum"));
|
||||
|
||||
System.out.println("\n=== Aggregate: Sum by Activity ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAvgByRegion() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"function", "avg",
|
||||
"by", List.of("region")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 2 rows (us-east and us-west)
|
||||
assertEquals(2, result.rowCount());
|
||||
assertTrue(result.columns().contains("region"));
|
||||
assertTrue(result.columns().contains("avg"));
|
||||
|
||||
System.out.println("\n=== Aggregate: Avg by Region ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMaxByHost() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"function", "max",
|
||||
"by", List.of("host")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 3 rows (server1, server2, server3)
|
||||
assertEquals(3, result.rowCount(), "Should have 3 hosts");
|
||||
|
||||
System.out.println("\n=== Aggregate: Max by Host ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCountByMultipleLabels() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"function", "count",
|
||||
"by", List.of("activity", "region")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 4 rows (2 activities * 2 regions)
|
||||
assertEquals(4, result.rowCount(), "Should have 4 combinations");
|
||||
assertEquals(3, result.columns().size());
|
||||
assertTrue(result.columns().contains("activity"));
|
||||
assertTrue(result.columns().contains("region"));
|
||||
assertTrue(result.columns().contains("count"));
|
||||
|
||||
System.out.println("\n=== Aggregate: Count by Activity and Region ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSumWithLabelFilter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"function", "sum",
|
||||
"labels", Map.of("env", "prod")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
|
||||
System.out.println("\n=== Aggregate: Sum (env=prod only) ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSumExamplesDatabase() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"function", "sum",
|
||||
"by", List.of("status")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 3 rows (200, 404, 500)
|
||||
assertEquals(3, result.rowCount());
|
||||
|
||||
System.out.println("\n=== Aggregate: Sum by Status Code ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("function", "sum")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingFunction() {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationInvalidFunction() {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "function", "invalid")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationValidFunctions() throws Exception {
|
||||
AggregateCommand command = new AggregateCommand();
|
||||
|
||||
command.validate(Map.of("metric", "test", "function", "sum"));
|
||||
command.validate(Map.of("metric", "test", "function", "avg"));
|
||||
command.validate(Map.of("metric", "test", "function", "max"));
|
||||
command.validate(Map.of("metric", "test", "function", "min"));
|
||||
command.validate(Map.of("metric", "test", "function", "count"));
|
||||
}
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class IncreaseCommandTest {
|
||||
|
||||
@Test
|
||||
void testIncreaseLinearPattern() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "patterns_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("pattern", "linear")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 1 row for the linear pattern label set
|
||||
assertEquals(1, result.rowCount(), "Should have 1 result for linear pattern");
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
double increase = (double) row.get("increase");
|
||||
|
||||
// Linear pattern: 120 snapshots * 10 increment = 1190 total increase
|
||||
assertEquals(1190.0, increase, 1.0, "Total increase should be ~1190");
|
||||
|
||||
System.out.println("\n=== Increase Query: Linear Pattern ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIncreaseExponentialPattern() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "patterns_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("pattern", "exponential")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
double increase = (double) row.get("increase");
|
||||
|
||||
// Exponential: 100 * (1.01^120) ≈ 230
|
||||
assertTrue(increase > 200, "Exponential increase should be > 200");
|
||||
assertTrue(increase < 300, "Exponential increase should be < 300");
|
||||
|
||||
System.out.println("\n=== Increase Query: Exponential Pattern ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIncreaseStepPattern() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "patterns_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("pattern", "step")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
double increase = (double) row.get("increase");
|
||||
|
||||
// Step pattern: starts at 100, ends at 1200 (12 steps * 100)
|
||||
assertEquals(1100.0, increase, 1.0, "Step increase should be 1100");
|
||||
|
||||
System.out.println("\n=== Increase Query: Step Pattern ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIncreaseExamplesDatabase() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("method", "GET", "endpoint", "/users", "status", "200")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
double increase = (double) row.get("increase");
|
||||
|
||||
// From 0 to 11000
|
||||
assertEquals(11000.0, increase, 0.1);
|
||||
|
||||
System.out.println("\n=== Increase Query: Examples DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIncreaseMultipleLabels() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("status", "200")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 3 label sets (GET /users, GET /products, POST /users)
|
||||
assertEquals(3, result.rowCount(), "Should have 3 label sets with status=200");
|
||||
|
||||
System.out.println("\n=== Increase Query: All Successful Requests ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("window", "5m")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingWindow() {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationBothWindowAndLast() {
|
||||
IncreaseCommand command = new IncreaseCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "window", "5m", "last", "10m")));
|
||||
}
|
||||
}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class InstantCommandTest {
|
||||
|
||||
@Test
|
||||
void testInstantQuerySimpleCounter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
InstantCommand command = new InstantCommand();
|
||||
|
||||
Map<String, Object> params = Map.of("metric", "activity_ops_total");
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 2 rows (read and write activities)
|
||||
assertEquals(2, result.rowCount(), "Expected 2 label sets");
|
||||
|
||||
// Check columns
|
||||
assertEquals(3, result.columns().size());
|
||||
assertTrue(result.columns().contains("timestamp"));
|
||||
assertTrue(result.columns().contains("value"));
|
||||
assertTrue(result.columns().contains("labels"));
|
||||
|
||||
// Verify we got both read and write
|
||||
boolean hasRead = false;
|
||||
boolean hasWrite = false;
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
double value = (double) row.get("value");
|
||||
|
||||
if (labels.contains("activity=read")) {
|
||||
hasRead = true;
|
||||
assertEquals(90.0, value, 0.01, "Expected final read value to be 90");
|
||||
} else if (labels.contains("activity=write")) {
|
||||
hasWrite = true;
|
||||
assertEquals(135.0, value, 0.01, "Expected final write value to be 135");
|
||||
}
|
||||
}
|
||||
|
||||
assertTrue(hasRead, "Should have read activity");
|
||||
assertTrue(hasWrite, "Should have write activity");
|
||||
|
||||
// Print formatted output for visual verification
|
||||
System.out.println("\n=== Instant Query Result ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInstantQueryWithLabelFilter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
InstantCommand command = new InstantCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "activity_ops_total",
|
||||
"labels", Map.of("activity", "read")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return only 1 row (read activity)
|
||||
assertEquals(1, result.rowCount(), "Expected 1 label set with activity=read filter");
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
String labels = (String) row.get("labels");
|
||||
double value = (double) row.get("value");
|
||||
|
||||
assertTrue(labels.contains("activity=read"));
|
||||
assertEquals(90.0, value, 0.01);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInstantQueryMultiDimensional() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
InstantCommand command = new InstantCommand();
|
||||
|
||||
Map<String, Object> params = Map.of("metric", "requests_total");
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 24 rows (24 unique label combinations)
|
||||
assertEquals(24, result.rowCount(), "Expected 24 unique label sets");
|
||||
|
||||
System.out.println("\n=== Multi-Dimensional Instant Query ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInstantQueryWithMultipleLabelFilters() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
InstantCommand command = new InstantCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"labels", Map.of(
|
||||
"activity", "write",
|
||||
"region", "us-east",
|
||||
"env", "prod"
|
||||
)
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return only rows matching all 3 label filters
|
||||
// With 3 hosts, should get 3 results
|
||||
assertEquals(3, result.rowCount(),
|
||||
"Expected 3 label sets (one per host) matching activity=write, region=us-east, env=prod");
|
||||
|
||||
// Verify all results have the correct labels
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
assertTrue(labels.contains("activity=write"));
|
||||
assertTrue(labels.contains("region=us-east"));
|
||||
assertTrue(labels.contains("env=prod"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
InstantCommand command = new InstantCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationEmptyMetric() {
|
||||
InstantCommand command = new InstantCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationInvalidLabelsType() {
|
||||
InstantCommand command = new InstantCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "labels", "not a map")));
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class QuantileCommandTest {
|
||||
|
||||
@Test
|
||||
void testQuantileP95AllOperations() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "operation_latency",
|
||||
"quantile", 0.95
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should have p95 values for all operations across all snapshots
|
||||
assertTrue(result.rowCount() > 0, "Should have quantile results");
|
||||
|
||||
// Verify we have the expected columns
|
||||
assertEquals(4, result.columns().size());
|
||||
assertTrue(result.columns().contains("timestamp"));
|
||||
assertTrue(result.columns().contains("quantile"));
|
||||
assertTrue(result.columns().contains("quantile_value"));
|
||||
assertTrue(result.columns().contains("labels"));
|
||||
|
||||
// All quantiles should be 0.95
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
double q = (double) row.get("quantile");
|
||||
assertEquals(0.95, q, 0.001);
|
||||
}
|
||||
|
||||
System.out.println("\n=== Quantile Query: p95 All Operations ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQuantileP99SpecificOperation() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "operation_latency",
|
||||
"quantile", 0.99,
|
||||
"labels", Map.of("operation", "select")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should have p99 for select operation");
|
||||
|
||||
// All should be for select operation
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
assertTrue(labels.contains("operation=select"));
|
||||
}
|
||||
|
||||
System.out.println("\n=== Quantile Query: p99 Select Operation ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQuantileP50Median() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "operation_latency",
|
||||
"quantile", 0.50
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should have median values");
|
||||
|
||||
System.out.println("\n=== Quantile Query: p50 (Median) ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQuantileMultipleOperationsP95() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.LATENCY_TIMERS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "operation_latency",
|
||||
"quantile", 0.95
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should have results for all 4 operations (select, insert, update, delete)
|
||||
// across 6 snapshots = 24 total
|
||||
assertEquals(24, result.rowCount(), "Should have 24 quantile values (4 ops * 6 snapshots)");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("quantile", 0.95)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingQuantile() {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationQuantileOutOfRange() {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "quantile", -0.1)));
|
||||
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "quantile", 1.5)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationValidQuantiles() throws Exception {
|
||||
QuantileCommand command = new QuantileCommand();
|
||||
|
||||
command.validate(Map.of("metric", "test", "quantile", 0.0));
|
||||
command.validate(Map.of("metric", "test", "quantile", 0.5));
|
||||
command.validate(Map.of("metric", "test", "quantile", 1.0));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class RangeCommandTest {
|
||||
|
||||
@Test
|
||||
void testRangeQueryAllData() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RangeCommand command = new RangeCommand();
|
||||
|
||||
// Query all data using a very large window
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "activity_ops_total",
|
||||
"window", "1h"
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return all 20 samples (10 snapshots × 2 label sets)
|
||||
assertEquals(20, result.rowCount(), "Should return all samples");
|
||||
|
||||
// Verify columns
|
||||
assertEquals(3, result.columns().size());
|
||||
assertTrue(result.columns().contains("timestamp"));
|
||||
assertTrue(result.columns().contains("value"));
|
||||
assertTrue(result.columns().contains("labels"));
|
||||
|
||||
System.out.println("\n=== Range Query All Data ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRangeQueryWithLabelFilter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RangeCommand command = new RangeCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "activity_ops_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("activity", "read")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 10 samples (just read activity)
|
||||
assertEquals(10, result.rowCount(), "Should return only read activity samples");
|
||||
|
||||
// Verify all have activity=read
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
assertTrue(labels.contains("activity=read"), "All results should be activity=read");
|
||||
assertFalse(labels.contains("activity=write"), "Should not contain write");
|
||||
}
|
||||
|
||||
System.out.println("\n=== Range Query Filtered (read only) ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRangeQueryVerifyTimeSeries() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RangeCommand command = new RangeCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "activity_ops_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("activity", "read")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Verify values form expected linear sequence: 0, 10, 20, ..., 90
|
||||
List<Double> values = result.rows().stream()
|
||||
.map(row -> (Double) row.get("value"))
|
||||
.sorted()
|
||||
.toList();
|
||||
|
||||
assertEquals(10, values.size());
|
||||
for (int i = 0; i < 10; i++) {
|
||||
assertEquals(i * 10.0, values.get(i), 0.01,
|
||||
"Value at index " + i + " should be " + (i * 10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRangeQueryExamplesDatabase() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RangeCommand command = new RangeCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"window", "1h", // Get all 5 snapshots
|
||||
"labels", Map.of("method", "GET", "endpoint", "/users", "status", "200")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 5 snapshots
|
||||
assertEquals(5, result.rowCount(), "Should return all 5 snapshots");
|
||||
|
||||
// Verify values are monotonically increasing: 0, 1000, 2200, 5500, 11000
|
||||
List<Double> values = result.rows().stream()
|
||||
.map(row -> (Double) row.get("value"))
|
||||
.toList();
|
||||
|
||||
assertEquals(0.0, values.get(0), 0.01);
|
||||
assertEquals(1000.0, values.get(1), 0.01);
|
||||
assertEquals(2200.0, values.get(2), 0.01);
|
||||
assertEquals(5500.0, values.get(3), 0.01);
|
||||
assertEquals(11000.0, values.get(4), 0.01);
|
||||
|
||||
System.out.println("\n=== Range Query Examples DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRangeQueryAlternativeLastSyntax() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RangeCommand command = new RangeCommand();
|
||||
|
||||
// Test --last as alternative to --window
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "activity_ops_total",
|
||||
"last", "1h"
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(20, result.rowCount(), "Should return all samples with --last");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
RangeCommand command = new RangeCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("window", "5m")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingTimeRange() {
|
||||
RangeCommand command = new RangeCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMultipleTimeRangeMethods() {
|
||||
RangeCommand command = new RangeCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "window", "5m", "last", "10m")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class RateCommandTest {
|
||||
|
||||
@Test
|
||||
void testRateLinearPattern() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RateCommand command = new RateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "patterns_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("pattern", "linear")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Linear pattern: increments by 10 every ~50ms
|
||||
// Expected rate: 10 / 0.05 = 200 ops/sec (approximately)
|
||||
assertTrue(result.rowCount() > 0, "Should have rate calculations");
|
||||
|
||||
// Get all calculated rates
|
||||
List<Double> rates = result.rows().stream()
|
||||
.map(row -> (Double) row.get("rate_per_sec"))
|
||||
.filter(rate -> rate != null && !rate.isNaN())
|
||||
.toList();
|
||||
|
||||
assertTrue(rates.size() > 0, "Should have valid rates");
|
||||
|
||||
// For linear pattern, all rates should be relatively constant
|
||||
// (within reasonable variance due to timing)
|
||||
double avgRate = rates.stream().mapToDouble(Double::doubleValue).average().orElse(0);
|
||||
assertTrue(avgRate > 0, "Average rate should be positive");
|
||||
|
||||
System.out.println("\n=== Rate Query: Linear Pattern ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
System.out.println("Average rate: " + avgRate + " ops/sec");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRateExponentialPattern() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RateCommand command = new RateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "patterns_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("pattern", "exponential")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should have rate calculations");
|
||||
|
||||
// Exponential pattern: rate should increase over time
|
||||
List<Double> rates = result.rows().stream()
|
||||
.map(row -> (Double) row.get("rate_per_sec"))
|
||||
.filter(rate -> rate != null && !rate.isNaN())
|
||||
.toList();
|
||||
|
||||
assertTrue(rates.size() > 10, "Should have many rate points");
|
||||
|
||||
// Verify rates are generally increasing (exponential growth)
|
||||
// Compare first quarter to last quarter
|
||||
int quarterSize = rates.size() / 4;
|
||||
double firstQuarterAvg = rates.subList(0, quarterSize).stream()
|
||||
.mapToDouble(Double::doubleValue).average().orElse(0);
|
||||
double lastQuarterAvg = rates.subList(rates.size() - quarterSize, rates.size()).stream()
|
||||
.mapToDouble(Double::doubleValue).average().orElse(0);
|
||||
|
||||
assertTrue(lastQuarterAvg > firstQuarterAvg,
|
||||
"Exponential pattern should have increasing rate over time");
|
||||
|
||||
System.out.println("\n=== Rate Query: Exponential Pattern ===");
|
||||
System.out.println("First quarter avg: " + firstQuarterAvg + " ops/sec");
|
||||
System.out.println("Last quarter avg: " + lastQuarterAvg + " ops/sec");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRateStepPattern() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.RATE_CALCULATIONS);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RateCommand command = new RateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "patterns_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("pattern", "step")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should have rate calculations");
|
||||
|
||||
// Step pattern: rate should be 0 within steps, high at step boundaries
|
||||
List<Double> rates = result.rows().stream()
|
||||
.map(row -> (Double) row.get("rate_per_sec"))
|
||||
.filter(rate -> rate != null && !rate.isNaN())
|
||||
.toList();
|
||||
|
||||
// Most rates should be either 0 (flat) or very high (step jump)
|
||||
long zeroRates = rates.stream().filter(r -> Math.abs(r) < 1.0).count();
|
||||
long highRates = rates.stream().filter(r -> r > 100.0).count();
|
||||
|
||||
assertTrue(zeroRates > 0 || highRates > 0,
|
||||
"Step pattern should have either zero or high rates");
|
||||
|
||||
System.out.println("\n=== Rate Query: Step Pattern ===");
|
||||
System.out.println("Zero rates: " + zeroRates + ", High rates: " + highRates);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRateExamplesDatabase() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
RateCommand command = new RateCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"window", "1h",
|
||||
"labels", Map.of("method", "GET", "endpoint", "/users", "status", "200")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should have 4 rate calculations (5 snapshots - 1 for first point)
|
||||
assertEquals(4, result.rowCount(), "Should have 4 rate points");
|
||||
|
||||
System.out.println("\n=== Rate Query: Examples DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
RateCommand command = new RateCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("window", "5m")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingWindow() {
|
||||
RateCommand command = new RateCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationBothWindowAndLast() {
|
||||
RateCommand command = new RateCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "window", "5m", "last", "10m")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationWindowAllowed() throws Exception {
|
||||
RateCommand command = new RateCommand();
|
||||
command.validate(Map.of("metric", "test", "window", "5m"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationLastAllowed() throws Exception {
|
||||
RateCommand command = new RateCommand();
|
||||
command.validate(Map.of("metric", "test", "last", "5m"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class RatioCommandTest {
|
||||
|
||||
@Test
|
||||
void testRatioCommandExists() throws Exception {
|
||||
// Basic smoke test - just verify command can be created and validated
|
||||
RatioCommand command = new RatioCommand();
|
||||
assertEquals("ratio", command.getName());
|
||||
|
||||
// Valid params should pass validation
|
||||
command.validate(Map.of(
|
||||
"numerator", "metric1_total",
|
||||
"denominator", "metric2_total"
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testValidationMissingNumerator() {
|
||||
RatioCommand command = new RatioCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("denominator", "total")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingDenominator() {
|
||||
RatioCommand command = new RatioCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("numerator", "errors")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationEmptyMetrics() {
|
||||
RatioCommand command = new RatioCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("numerator", "", "denominator", "total")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationValid() throws Exception {
|
||||
RatioCommand command = new RatioCommand();
|
||||
command.validate(Map.of("numerator", "errors", "denominator", "total"));
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class SessionCommandTest {
|
||||
|
||||
@Test
|
||||
void testSessionInfo() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SessionCommand command = new SessionCommand();
|
||||
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
// Should return 1 row with session info
|
||||
assertEquals(1, result.rowCount());
|
||||
|
||||
// Verify columns
|
||||
assertEquals(6, result.columns().size());
|
||||
assertTrue(result.columns().contains("first_snapshot"));
|
||||
assertTrue(result.columns().contains("last_snapshot"));
|
||||
assertTrue(result.columns().contains("duration"));
|
||||
assertTrue(result.columns().contains("total_snapshots"));
|
||||
assertTrue(result.columns().contains("total_samples"));
|
||||
assertTrue(result.columns().contains("avg_interval"));
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
|
||||
// Verify we have 5 snapshots
|
||||
int snapshots = (int) row.get("total_snapshots");
|
||||
assertEquals(5, snapshots, "Should have 5 snapshots");
|
||||
|
||||
// Should have 35 total samples
|
||||
int samples = (int) row.get("total_samples");
|
||||
assertEquals(35, samples, "Should have 35 samples");
|
||||
|
||||
System.out.println("\n=== Session Command ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSessionInfoSimpleCounter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SessionCommand command = new SessionCommand();
|
||||
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
int snapshots = (int) row.get("total_snapshots");
|
||||
int samples = (int) row.get("total_samples");
|
||||
|
||||
assertEquals(10, snapshots, "Should have 10 snapshots");
|
||||
assertEquals(20, samples, "Should have 20 samples");
|
||||
|
||||
System.out.println("\n=== Session Info: Simple Counter DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class SqlCommandTest {
|
||||
|
||||
@Test
|
||||
void testSqlQueryMetricFamilies() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"query", "SELECT name, type FROM metric_family ORDER BY name"
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should have at least 1 metric family
|
||||
assertTrue(result.rowCount() >= 1, "Should have metric families");
|
||||
|
||||
// Verify columns
|
||||
assertEquals(2, result.columns().size());
|
||||
assertTrue(result.columns().contains("name"));
|
||||
assertTrue(result.columns().contains("type"));
|
||||
|
||||
System.out.println("\n=== SQL Query: Metric Families ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSqlQuerySampleCounts() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"query", "SELECT COUNT(*) as total_samples FROM sample_value"
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
assertEquals(1, result.columns().size());
|
||||
assertEquals("total_samples", result.columns().get(0));
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
long count = ((Number) row.get("total_samples")).longValue();
|
||||
assertEquals(20, count, "Should have 20 samples");
|
||||
|
||||
System.out.println("\n=== SQL Query: Sample Count ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSqlQueryWithCTE() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
String sql = """
|
||||
WITH latest AS (
|
||||
SELECT MAX(timestamp_ms) as max_ts FROM sample_value
|
||||
)
|
||||
SELECT COUNT(*) as latest_count
|
||||
FROM sample_value sv, latest
|
||||
WHERE sv.timestamp_ms = latest.max_ts
|
||||
""";
|
||||
|
||||
Map<String, Object> params = Map.of("query", sql);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount());
|
||||
long count = ((Number) result.rows().get(0).get("latest_count")).longValue();
|
||||
assertEquals(2, count, "Should have 2 samples at latest timestamp");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSqlQueryPragma() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"query", "PRAGMA table_info(metric_family)"
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return schema information
|
||||
assertTrue(result.rowCount() > 0, "Should return table info");
|
||||
|
||||
System.out.println("\n=== SQL Query: Table Schema ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingQuery() {
|
||||
SqlCommand command = new SqlCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationEmptyQuery() {
|
||||
SqlCommand command = new SqlCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("query", "")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationNonSelectQuery() {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
// INSERT should be blocked
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("query", "INSERT INTO metric_family ...")));
|
||||
|
||||
// UPDATE should be blocked
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("query", "UPDATE sample_value SET ...")));
|
||||
|
||||
// DELETE should be blocked
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("query", "DELETE FROM sample_value")));
|
||||
|
||||
// DROP should be blocked
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("query", "DROP TABLE metric_family")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationAllowedQueries() throws Exception {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
// SELECT should be allowed
|
||||
command.validate(Map.of("query", "SELECT * FROM metric_family"));
|
||||
|
||||
// WITH (CTE) should be allowed
|
||||
command.validate(Map.of("query", "WITH cte AS (SELECT 1) SELECT * FROM cte"));
|
||||
|
||||
// PRAGMA should be allowed
|
||||
command.validate(Map.of("query", "PRAGMA table_info(metric_family)"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSqlQueryExamples() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SqlCommand command = new SqlCommand();
|
||||
|
||||
// Example: Get all metrics with their latest values
|
||||
String sql = """
|
||||
SELECT
|
||||
sn.sample,
|
||||
sv.value,
|
||||
datetime(sv.timestamp_ms/1000, 'unixepoch') as captured_at
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
WHERE sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
ORDER BY sn.sample, sv.value DESC
|
||||
""";
|
||||
|
||||
Map<String, Object> params = Map.of("query", sql);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should return results");
|
||||
|
||||
System.out.println("\n=== SQL Query: All Latest Values ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class SummaryCommandTest {
|
||||
|
||||
@Test
|
||||
void testSummaryExamplesDb() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SummaryCommand command = new SummaryCommand();
|
||||
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
// Should have multiple sections
|
||||
assertTrue(result.rowCount() > 0, "Should have summary rows");
|
||||
|
||||
// Verify columns
|
||||
assertEquals(4, result.columns().size());
|
||||
assertTrue(result.columns().contains("section"));
|
||||
assertTrue(result.columns().contains("metric"));
|
||||
assertTrue(result.columns().contains("value"));
|
||||
assertTrue(result.columns().contains("details"));
|
||||
|
||||
// Should have SESSION section
|
||||
boolean hasSession = result.rows().stream()
|
||||
.anyMatch(row -> "SESSION".equals(row.get("section")));
|
||||
assertTrue(hasSession, "Should have SESSION section");
|
||||
|
||||
System.out.println("\n=== Summary: Examples DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSummarySimpleCounter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.SIMPLE_COUNTER);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SummaryCommand command = new SummaryCommand();
|
||||
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should have summary rows");
|
||||
|
||||
System.out.println("\n=== Summary: Simple Counter DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSummaryMultiDimensional() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
SummaryCommand command = new SummaryCommand();
|
||||
|
||||
QueryResult result = command.execute(conn, Map.of());
|
||||
|
||||
assertTrue(result.rowCount() > 0, "Should have summary rows");
|
||||
|
||||
System.out.println("\n=== Summary: Multi-Dimensional DB ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.commands;
|
||||
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.InvalidQueryException;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class TopKCommandTest {
|
||||
|
||||
@Test
|
||||
void testTop3() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
TopKCommand command = new TopKCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"n", 3
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return exactly 3 rows
|
||||
assertEquals(3, result.rowCount(), "Should return top 3");
|
||||
|
||||
// Verify columns
|
||||
assertEquals(2, result.columns().size());
|
||||
assertTrue(result.columns().contains("value"));
|
||||
assertTrue(result.columns().contains("labels"));
|
||||
|
||||
// First row should have highest value (11000)
|
||||
double firstValue = (double) result.rows().get(0).get("value");
|
||||
assertEquals(11000.0, firstValue, 0.1, "Top value should be 11000");
|
||||
|
||||
System.out.println("\n=== TopK Query: Top 3 ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTop1() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
TopKCommand command = new TopKCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"n", 1
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
assertEquals(1, result.rowCount(), "Should return top 1");
|
||||
|
||||
double value = (double) result.rows().get(0).get("value");
|
||||
assertEquals(11000.0, value, 0.1);
|
||||
|
||||
System.out.println("\n=== TopK Query: Top 1 ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTop5WithLabelFilter() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
TopKCommand command = new TopKCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "api_requests_total",
|
||||
"n", 5,
|
||||
"labels", Map.of("status", "200")
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return 3 rows (only 3 label sets with status=200)
|
||||
assertEquals(3, result.rowCount(), "Should return 3 (all status=200)");
|
||||
|
||||
// All should have status=200
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
assertTrue(labels.contains("status=200"));
|
||||
}
|
||||
|
||||
System.out.println("\n=== TopK Query: Top 5 (filtered by status=200) ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testTop10MultiDimensional() throws Exception {
|
||||
Path dbPath = TestDatabaseLoader.getDatabase(TestDatabaseLoader.TestDatabase.MULTI_DIMENSIONAL);
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(dbPath)) {
|
||||
TopKCommand command = new TopKCommand();
|
||||
|
||||
Map<String, Object> params = Map.of(
|
||||
"metric", "requests_total",
|
||||
"n", 10
|
||||
);
|
||||
QueryResult result = command.execute(conn, params);
|
||||
|
||||
// Should return exactly 10 rows
|
||||
assertEquals(10, result.rowCount(), "Should return top 10");
|
||||
|
||||
// Verify values are in descending order
|
||||
double previousValue = Double.MAX_VALUE;
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
double value = (double) row.get("value");
|
||||
assertTrue(value <= previousValue, "Values should be in descending order");
|
||||
previousValue = value;
|
||||
}
|
||||
|
||||
System.out.println("\n=== TopK Query: Top 10 from Multi-Dimensional ===");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingMetric() {
|
||||
TopKCommand command = new TopKCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("n", 5)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationMissingN() {
|
||||
TopKCommand command = new TopKCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationNegativeN() {
|
||||
TopKCommand command = new TopKCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "n", -1)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationZeroN() {
|
||||
TopKCommand command = new TopKCommand();
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
command.validate(Map.of("metric", "test", "n", 0)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testValidationValidN() throws Exception {
|
||||
TopKCommand command = new TopKCommand();
|
||||
command.validate(Map.of("metric", "test", "n", 1));
|
||||
command.validate(Map.of("metric", "test", "n", 100));
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.integration;
|
||||
|
||||
import io.nosqlbench.nb.mql.commands.InstantCommand;
|
||||
import io.nosqlbench.nb.mql.commands.RangeCommand;
|
||||
import io.nosqlbench.nb.mql.format.TableFormatter;
|
||||
import io.nosqlbench.nb.mql.query.QueryResult;
|
||||
import io.nosqlbench.nb.mql.schema.MetricsDatabaseReader;
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Validates that the examples in EXAMPLES.md produce the expected output.
|
||||
* This test parses the markdown file, executes the commands, and verifies
|
||||
* the results match the documented expectations.
|
||||
*/
|
||||
class ExampleDocumentationTest {
|
||||
|
||||
private static Path examplesDb;
|
||||
private static List<Example> examples;
|
||||
|
||||
@BeforeAll
|
||||
static void loadExamples() throws IOException {
|
||||
// Load the examples database
|
||||
examplesDb = TestDatabaseLoader.getDatabase("examples.db");
|
||||
|
||||
// Parse all example markdown files from classpath
|
||||
examples = new ArrayList<>();
|
||||
|
||||
String[] exampleFiles = {
|
||||
"mql-instant-queries.md",
|
||||
"mql-range-queries.md"
|
||||
// Add more as they are created:
|
||||
// "mql-rate-queries.md",
|
||||
// "mql-aggregation-queries.md",
|
||||
// etc.
|
||||
};
|
||||
|
||||
for (String exampleFile : exampleFiles) {
|
||||
String content = new String(
|
||||
ExampleDocumentationTest.class.getClassLoader()
|
||||
.getResourceAsStream(exampleFile)
|
||||
.readAllBytes()
|
||||
);
|
||||
examples.addAll(parseExamples(content));
|
||||
System.out.println("Loaded examples from " + exampleFile);
|
||||
}
|
||||
|
||||
System.out.println("Total: " + examples.size() + " examples loaded");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExample1_InstantQueryAllLabels() throws Exception {
|
||||
Example example = findExample("Example 1");
|
||||
assertNotNull(example, "Example 1 should exist");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
|
||||
QueryResult result = executeCommand(conn, example.command);
|
||||
|
||||
// Verify row count
|
||||
assertEquals(5, result.rowCount(), "Example 1 should return 5 label sets");
|
||||
|
||||
// Verify all values are present in the result
|
||||
Map<String, Double> expectedValues = Map.of(
|
||||
"endpoint=/users, method=GET, status=200", 11000.0,
|
||||
"endpoint=/products, method=GET, status=200", 8500.0,
|
||||
"endpoint=/users, method=POST, status=200", 1500.0,
|
||||
"endpoint=/users, method=GET, status=404", 75.0,
|
||||
"endpoint=/users, method=GET, status=500", 15.0
|
||||
);
|
||||
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
double value = (double) row.get("value");
|
||||
|
||||
boolean matched = false;
|
||||
for (Map.Entry<String, Double> expected : expectedValues.entrySet()) {
|
||||
if (labelsMatch(labels, expected.getKey())) {
|
||||
assertEquals(expected.getValue(), value, 0.01,
|
||||
"Value mismatch for labels: " + labels);
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assertTrue(matched, "Unexpected label set: " + labels);
|
||||
}
|
||||
|
||||
System.out.println("Example 1 output:");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExample2_InstantQueryWithFilter() throws Exception {
|
||||
Example example = findExample("Example 2");
|
||||
assertNotNull(example, "Example 2 should exist");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
|
||||
QueryResult result = executeCommand(conn, example.command);
|
||||
|
||||
// Should return exactly 1 row
|
||||
assertEquals(1, result.rowCount(), "Filtered query should return 1 row");
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
double value = (double) row.get("value");
|
||||
String labels = (String) row.get("labels");
|
||||
|
||||
assertEquals(11000.0, value, 0.01);
|
||||
assertTrue(labels.contains("method=GET"));
|
||||
assertTrue(labels.contains("endpoint=/users"));
|
||||
assertTrue(labels.contains("status=200"));
|
||||
|
||||
System.out.println("Example 2 output:");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExample3_InstantQueryErrors() throws Exception {
|
||||
Example example = findExample("Example 3");
|
||||
assertNotNull(example, "Example 3 should exist");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
|
||||
QueryResult result = executeCommand(conn, example.command);
|
||||
|
||||
assertEquals(1, result.rowCount(), "Should return 1 error metric");
|
||||
|
||||
Map<String, Object> row = result.rows().get(0);
|
||||
double value = (double) row.get("value");
|
||||
String labels = (String) row.get("labels");
|
||||
|
||||
assertEquals(15.0, value, 0.01);
|
||||
assertTrue(labels.contains("status=500"));
|
||||
|
||||
System.out.println("Example 3 output:");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExample4_InstantQuerySuccessful() throws Exception {
|
||||
Example example = findExample("Example 4");
|
||||
assertNotNull(example, "Example 4 should exist");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
|
||||
QueryResult result = executeCommand(conn, example.command);
|
||||
|
||||
assertEquals(3, result.rowCount(), "Should return 3 successful request types");
|
||||
|
||||
// Verify all have status=200
|
||||
for (Map<String, Object> row : result.rows()) {
|
||||
String labels = (String) row.get("labels");
|
||||
assertTrue(labels.contains("status=200"), "All results should have status=200");
|
||||
}
|
||||
|
||||
System.out.println("Example 4 output:");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExample5_RangeQueryTimeSeries() throws Exception {
|
||||
Example example = findExample("Example 5");
|
||||
assertNotNull(example, "Example 5 should exist");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
|
||||
QueryResult result = executeCommand(conn, example.command);
|
||||
|
||||
// Should return 5 snapshots for the single label set
|
||||
assertEquals(5, result.rowCount(), "Should return 5 time-series points");
|
||||
|
||||
// Verify values are: 0, 1000, 2200, 5500, 11000
|
||||
List<Double> values = result.rows().stream()
|
||||
.map(row -> (Double) row.get("value"))
|
||||
.toList();
|
||||
|
||||
assertEquals(0.0, values.get(0), 0.01);
|
||||
assertEquals(1000.0, values.get(1), 0.01);
|
||||
assertEquals(2200.0, values.get(2), 0.01);
|
||||
assertEquals(5500.0, values.get(3), 0.01);
|
||||
assertEquals(11000.0, values.get(4), 0.01);
|
||||
|
||||
System.out.println("Example 5 output:");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExample6_RangeQueryErrors() throws Exception {
|
||||
Example example = findExample("Example 6");
|
||||
assertNotNull(example, "Example 6 should exist");
|
||||
|
||||
try (Connection conn = MetricsDatabaseReader.connect(examplesDb)) {
|
||||
QueryResult result = executeCommand(conn, example.command);
|
||||
|
||||
// Should return 5 snapshots for error metric
|
||||
assertEquals(5, result.rowCount(), "Should return 5 time-series points for errors");
|
||||
|
||||
// Verify error values: 0, 2, 4, 8, 15
|
||||
List<Double> values = result.rows().stream()
|
||||
.map(row -> (Double) row.get("value"))
|
||||
.toList();
|
||||
|
||||
assertEquals(0.0, values.get(0), 0.01);
|
||||
assertEquals(2.0, values.get(1), 0.01);
|
||||
assertEquals(4.0, values.get(2), 0.01);
|
||||
assertEquals(8.0, values.get(3), 0.01);
|
||||
assertEquals(15.0, values.get(4), 0.01);
|
||||
|
||||
System.out.println("Example 6 output:");
|
||||
System.out.println(new TableFormatter().format(result));
|
||||
}
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
private static QueryResult executeCommand(Connection conn, String commandStr) throws Exception {
|
||||
// Parse command string: "instant --metric <name> [--labels key=val,...]"
|
||||
Map<String, Object> params = parseCommandParams(commandStr);
|
||||
|
||||
if (commandStr.startsWith("instant")) {
|
||||
InstantCommand command = new InstantCommand();
|
||||
return command.execute(conn, params);
|
||||
} else if (commandStr.startsWith("range")) {
|
||||
RangeCommand command = new RangeCommand();
|
||||
return command.execute(conn, params);
|
||||
}
|
||||
|
||||
throw new UnsupportedOperationException("Command not yet implemented: " + commandStr);
|
||||
}
|
||||
|
||||
private static Map<String, Object> parseCommandParams(String commandStr) {
|
||||
Map<String, Object> params = new LinkedHashMap<>();
|
||||
|
||||
// Extract --metric value
|
||||
Pattern metricPattern = Pattern.compile("--metric\\s+(\\S+)");
|
||||
Matcher metricMatcher = metricPattern.matcher(commandStr);
|
||||
if (metricMatcher.find()) {
|
||||
params.put("metric", metricMatcher.group(1));
|
||||
}
|
||||
|
||||
// Extract --window value
|
||||
Pattern windowPattern = Pattern.compile("--window\\s+(\\S+)");
|
||||
Matcher windowMatcher = windowPattern.matcher(commandStr);
|
||||
if (windowMatcher.find()) {
|
||||
params.put("window", windowMatcher.group(1));
|
||||
}
|
||||
|
||||
// Extract --last value
|
||||
Pattern lastPattern = Pattern.compile("--last\\s+(\\S+)");
|
||||
Matcher lastMatcher = lastPattern.matcher(commandStr);
|
||||
if (lastMatcher.find()) {
|
||||
params.put("last", lastMatcher.group(1));
|
||||
}
|
||||
|
||||
// Extract --labels key=val,key=val
|
||||
Pattern labelsPattern = Pattern.compile("--labels\\s+(\\S+)");
|
||||
Matcher labelsMatcher = labelsPattern.matcher(commandStr);
|
||||
if (labelsMatcher.find()) {
|
||||
String labelsStr = labelsMatcher.group(1);
|
||||
Map<String, String> labels = new LinkedHashMap<>();
|
||||
for (String pair : labelsStr.split(",")) {
|
||||
String[] kv = pair.split("=", 2);
|
||||
if (kv.length == 2) {
|
||||
labels.put(kv[0], kv[1]);
|
||||
}
|
||||
}
|
||||
params.put("labels", labels);
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
private static boolean labelsMatch(String actual, String expected) {
|
||||
// Normalize: remove spaces, sort key=value pairs
|
||||
String[] actualPairs = actual.replace(" ", "").split(",");
|
||||
String[] expectedPairs = expected.replace(" ", "").split(",");
|
||||
|
||||
if (actualPairs.length != expectedPairs.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
java.util.Arrays.sort(actualPairs);
|
||||
java.util.Arrays.sort(expectedPairs);
|
||||
|
||||
return java.util.Arrays.equals(actualPairs, expectedPairs);
|
||||
}
|
||||
|
||||
private static Example findExample(String exampleName) {
|
||||
for (Example example : examples) {
|
||||
if (example.name.contains(exampleName)) {
|
||||
return example;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<Example> parseExamples(String markdown) {
|
||||
List<Example> result = new ArrayList<>();
|
||||
|
||||
// Pattern to match example sections
|
||||
Pattern examplePattern = Pattern.compile(
|
||||
"##\\s+(Example \\d+:[^\\n]+).*?" +
|
||||
"\\*\\*Command:\\*\\*\\s*```([^`]+)```",
|
||||
Pattern.DOTALL
|
||||
);
|
||||
|
||||
Matcher matcher = examplePattern.matcher(markdown);
|
||||
while (matcher.find()) {
|
||||
String name = matcher.group(1).trim();
|
||||
String command = matcher.group(2).trim();
|
||||
result.add(new Example(name, command));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private record Example(String name, String command) {}
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.query;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class TimeWindowParserTest {
|
||||
|
||||
@Test
|
||||
void testParseSeconds() throws InvalidQueryException {
|
||||
assertEquals(5000L, TimeWindowParser.parseToMillis("5s"));
|
||||
assertEquals(30000L, TimeWindowParser.parseToMillis("30s"));
|
||||
assertEquals(1000L, TimeWindowParser.parseToMillis("1s"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseMinutes() throws InvalidQueryException {
|
||||
assertEquals(60000L, TimeWindowParser.parseToMillis("1m"));
|
||||
assertEquals(300000L, TimeWindowParser.parseToMillis("5m"));
|
||||
assertEquals(900000L, TimeWindowParser.parseToMillis("15m"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseHours() throws InvalidQueryException {
|
||||
assertEquals(3600000L, TimeWindowParser.parseToMillis("1h"));
|
||||
assertEquals(7200000L, TimeWindowParser.parseToMillis("2h"));
|
||||
assertEquals(86400000L, TimeWindowParser.parseToMillis("24h"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseDays() throws InvalidQueryException {
|
||||
assertEquals(86400000L, TimeWindowParser.parseToMillis("1d"));
|
||||
assertEquals(604800000L, TimeWindowParser.parseToMillis("7d"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseWeeks() throws InvalidQueryException {
|
||||
assertEquals(604800000L, TimeWindowParser.parseToMillis("1w"));
|
||||
assertEquals(1209600000L, TimeWindowParser.parseToMillis("2w"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseYears() throws InvalidQueryException {
|
||||
assertEquals(31536000000L, TimeWindowParser.parseToMillis("1y"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseDecimal() throws InvalidQueryException {
|
||||
assertEquals(1500L, TimeWindowParser.parseToMillis("1.5s"));
|
||||
assertEquals(90000L, TimeWindowParser.parseToMillis("1.5m"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCaseInsensitive() throws InvalidQueryException {
|
||||
assertEquals(5000L, TimeWindowParser.parseToMillis("5S"));
|
||||
assertEquals(300000L, TimeWindowParser.parseToMillis("5M"));
|
||||
assertEquals(3600000L, TimeWindowParser.parseToMillis("1H"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWhitespace() throws InvalidQueryException {
|
||||
assertEquals(5000L, TimeWindowParser.parseToMillis(" 5s "));
|
||||
assertEquals(300000L, TimeWindowParser.parseToMillis("\t5m\n"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInvalidFormat() {
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis("5"));
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis("s"));
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis("5x"));
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis("five seconds"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNullOrEmpty() {
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis(null));
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis(""));
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis(" "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNegativeValue() {
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis("-5s"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testZeroValue() {
|
||||
assertThrows(InvalidQueryException.class, () ->
|
||||
TimeWindowParser.parseToMillis("0s"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFormatMillis() {
|
||||
assertEquals("5s", TimeWindowParser.formatMillis(5000L));
|
||||
assertEquals("5m", TimeWindowParser.formatMillis(300000L));
|
||||
assertEquals("1h", TimeWindowParser.formatMillis(3600000L));
|
||||
assertEquals("1d", TimeWindowParser.formatMillis(86400000L));
|
||||
assertEquals("1w", TimeWindowParser.formatMillis(604800000L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFormatMillisNonExact() {
|
||||
// Should fall back to smallest unit that works
|
||||
assertEquals("1500ms", TimeWindowParser.formatMillis(1500L));
|
||||
assertEquals("90s", TimeWindowParser.formatMillis(90000L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRoundTrip() throws InvalidQueryException {
|
||||
String[] inputs = {"5s", "10m", "2h", "1d", "1w"};
|
||||
for (String input : inputs) {
|
||||
long millis = TimeWindowParser.parseToMillis(input);
|
||||
String formatted = TimeWindowParser.formatMillis(millis);
|
||||
assertEquals(input, formatted, "Round trip failed for: " + input);
|
||||
}
|
||||
}
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.schema;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class LabelSetResolverTest {
|
||||
|
||||
@Test
|
||||
void testEmptyLabels() {
|
||||
assertEquals("{}", LabelSetResolver.computeHash(Map.of()));
|
||||
assertEquals("{}", LabelSetResolver.computeHash(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleLabel() {
|
||||
Map<String, String> labels = Map.of("activity", "write");
|
||||
assertEquals("{activity=write}", LabelSetResolver.computeHash(labels));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMultipleLabels() {
|
||||
Map<String, String> labels = Map.of(
|
||||
"activity", "write",
|
||||
"host", "server1",
|
||||
"region", "us-east"
|
||||
);
|
||||
// Should be sorted alphabetically by key
|
||||
assertEquals("{activity=write,host=server1,region=us-east}",
|
||||
LabelSetResolver.computeHash(labels));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLabelsSortedAlphabetically() {
|
||||
// Insert in reverse order to test sorting
|
||||
Map<String, String> labels = new LinkedHashMap<>();
|
||||
labels.put("z_last", "value3");
|
||||
labels.put("b_second", "value2");
|
||||
labels.put("a_first", "value1");
|
||||
|
||||
String hash = LabelSetResolver.computeHash(labels);
|
||||
assertEquals("{a_first=value1,b_second=value2,z_last=value3}", hash);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSameLabelsDifferentOrder() {
|
||||
Map<String, String> labels1 = new LinkedHashMap<>();
|
||||
labels1.put("activity", "write");
|
||||
labels1.put("host", "server1");
|
||||
|
||||
Map<String, String> labels2 = new LinkedHashMap<>();
|
||||
labels2.put("host", "server1");
|
||||
labels2.put("activity", "write");
|
||||
|
||||
// Should produce same hash regardless of insertion order
|
||||
assertEquals(LabelSetResolver.computeHash(labels1),
|
||||
LabelSetResolver.computeHash(labels2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSpecialCharacters() {
|
||||
Map<String, String> labels = Map.of(
|
||||
"label", "value=with=equals",
|
||||
"another", "value,with,commas"
|
||||
);
|
||||
String hash = LabelSetResolver.computeHash(labels);
|
||||
assertTrue(hash.contains("label=value=with=equals"));
|
||||
assertTrue(hash.contains("another=value,with,commas"));
|
||||
}
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.testdata;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
/**
|
||||
* Generates the reference example database used in EXAMPLES.md documentation.
|
||||
* This database has specific, well-known timestamps and values that are referenced
|
||||
* in documentation and validated by ExampleDocumentationTest.
|
||||
*
|
||||
* The data represents a simplified e-commerce API with:
|
||||
* - Request counters (GET/POST operations on different endpoints)
|
||||
* - API latency timers
|
||||
* - HTTP status codes (200, 404, 500)
|
||||
*
|
||||
* Timeline:
|
||||
* - 2025-10-23 10:00:00Z - T+0: Initial state (zeros)
|
||||
* - 2025-10-23 10:01:00Z - T+1min: Normal traffic
|
||||
* - 2025-10-23 10:02:00Z - T+2min: Increased load
|
||||
* - 2025-10-23 10:05:00Z - T+5min: High load period
|
||||
* - 2025-10-23 10:10:00Z - T+10min: Back to normal
|
||||
*/
|
||||
public class ExampleDataGenerator {
|
||||
|
||||
/**
|
||||
* Generate examples.db - the reference database for documentation examples.
|
||||
*/
|
||||
public static void generateExamples(Path outputPath) throws Exception {
|
||||
MetricsDataBuilder.create(outputPath)
|
||||
// T+0: Initial state (10:00:00)
|
||||
.atTime("2025-10-23T10:00:00Z")
|
||||
.counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 0, "method", "GET", "endpoint", "/products", "status", "200")
|
||||
.counter("api_requests", 0, "method", "POST", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "404")
|
||||
.counter("api_requests", 0, "method", "GET", "endpoint", "/users", "status", "500")
|
||||
.timer("api_latency", new long[]{10, 12, 15, 20, 25}, "method", "GET", "endpoint", "/users")
|
||||
.timer("api_latency", new long[]{8, 10, 12, 14, 16}, "method", "GET", "endpoint", "/products")
|
||||
|
||||
// T+1min: Normal traffic (10:01:00)
|
||||
.atTime("2025-10-23T10:01:00Z")
|
||||
.counter("api_requests", 1000, "method", "GET", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 800, "method", "GET", "endpoint", "/products", "status", "200")
|
||||
.counter("api_requests", 150, "method", "POST", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 5, "method", "GET", "endpoint", "/users", "status", "404")
|
||||
.counter("api_requests", 2, "method", "GET", "endpoint", "/users", "status", "500")
|
||||
.timer("api_latency", new long[]{10, 12, 15, 18, 22, 25, 30}, "method", "GET", "endpoint", "/users")
|
||||
.timer("api_latency", new long[]{8, 10, 11, 13, 15, 18}, "method", "GET", "endpoint", "/products")
|
||||
|
||||
// T+2min: Increased load (10:02:00)
|
||||
.atTime("2025-10-23T10:02:00Z")
|
||||
.counter("api_requests", 2200, "method", "GET", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 1650, "method", "GET", "endpoint", "/products", "status", "200")
|
||||
.counter("api_requests", 320, "method", "POST", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 12, "method", "GET", "endpoint", "/users", "status", "404")
|
||||
.counter("api_requests", 4, "method", "GET", "endpoint", "/users", "status", "500")
|
||||
.timer("api_latency", new long[]{12, 15, 18, 22, 28, 35, 42}, "method", "GET", "endpoint", "/users")
|
||||
.timer("api_latency", new long[]{9, 11, 13, 15, 18, 22}, "method", "GET", "endpoint", "/products")
|
||||
|
||||
// T+5min: High load period (10:05:00)
|
||||
.atTime("2025-10-23T10:05:00Z")
|
||||
.counter("api_requests", 5500, "method", "GET", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 4200, "method", "GET", "endpoint", "/products", "status", "200")
|
||||
.counter("api_requests", 750, "method", "POST", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 35, "method", "GET", "endpoint", "/users", "status", "404")
|
||||
.counter("api_requests", 8, "method", "GET", "endpoint", "/users", "status", "500")
|
||||
.timer("api_latency", new long[]{15, 20, 25, 32, 45, 58, 75}, "method", "GET", "endpoint", "/users")
|
||||
.timer("api_latency", new long[]{10, 13, 16, 20, 25, 30}, "method", "GET", "endpoint", "/products")
|
||||
|
||||
// T+10min: Back to normal (10:10:00)
|
||||
.atTime("2025-10-23T10:10:00Z")
|
||||
.counter("api_requests", 11000, "method", "GET", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 8500, "method", "GET", "endpoint", "/products", "status", "200")
|
||||
.counter("api_requests", 1500, "method", "POST", "endpoint", "/users", "status", "200")
|
||||
.counter("api_requests", 75, "method", "GET", "endpoint", "/users", "status", "404")
|
||||
.counter("api_requests", 15, "method", "GET", "endpoint", "/users", "status", "500")
|
||||
.timer("api_latency", new long[]{11, 14, 17, 22, 28, 35, 45}, "method", "GET", "endpoint", "/users")
|
||||
.timer("api_latency", new long[]{9, 11, 13, 16, 19, 24}, "method", "GET", "endpoint", "/products")
|
||||
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method to generate the examples database.
|
||||
*/
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path examplesDb = Path.of("nb-apis/nb-mql-api/src/test/resources/testdata/examples.db");
|
||||
|
||||
System.out.println("Generating examples.db for documentation...");
|
||||
generateExamples(examplesDb);
|
||||
System.out.println("✓ Generated examples.db");
|
||||
System.out.println("Location: " + examplesDb.toAbsolutePath());
|
||||
}
|
||||
}
|
||||
+249
@@ -0,0 +1,249 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.testdata;
|
||||
|
||||
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
|
||||
import io.nosqlbench.nb.api.components.core.NBComponent;
|
||||
import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer;
|
||||
import io.nosqlbench.nb.api.engine.metrics.reporters.MetricInstanceFilter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.reporters.SqliteSnapshotReporter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.view.MetricsView;
|
||||
import io.nosqlbench.nb.api.labels.NBLabels;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Simplified utility for creating test metrics databases with specific timestamps.
|
||||
* Provides a fluent API for building predictable test data for documentation examples.
|
||||
*
|
||||
* Example:
|
||||
* <pre>
|
||||
* MetricsDataBuilder.create("examples.db")
|
||||
* .atTime("2025-10-23T10:00:00Z")
|
||||
* .counter("ops_total", 100, "activity", "read")
|
||||
* .counter("ops_total", 150, "activity", "write")
|
||||
* .atTime("2025-10-23T10:01:00Z")
|
||||
* .counter("ops_total", 120, "activity", "read")
|
||||
* .counter("ops_total", 180, "activity", "write")
|
||||
* .build();
|
||||
* </pre>
|
||||
*/
|
||||
public class MetricsDataBuilder {
|
||||
|
||||
private final Path outputPath;
|
||||
private final List<Snapshot> snapshots = new ArrayList<>();
|
||||
private Snapshot currentSnapshot;
|
||||
|
||||
private MetricsDataBuilder(Path outputPath) {
|
||||
this.outputPath = outputPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new metrics data builder for the specified database file.
|
||||
*
|
||||
* @param dbPath Path to the output database file
|
||||
* @return New builder instance
|
||||
*/
|
||||
public static MetricsDataBuilder create(String dbPath) {
|
||||
return new MetricsDataBuilder(Path.of(dbPath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new metrics data builder for the specified path.
|
||||
*
|
||||
* @param outputPath Path to the output database file
|
||||
* @return New builder instance
|
||||
*/
|
||||
public static MetricsDataBuilder create(Path outputPath) {
|
||||
return new MetricsDataBuilder(outputPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new snapshot at the specified timestamp (ISO-8601 format).
|
||||
*
|
||||
* @param timestamp ISO-8601 timestamp (e.g., "2025-10-23T10:00:00Z")
|
||||
* @return This builder for chaining
|
||||
*/
|
||||
public MetricsDataBuilder atTime(String timestamp) {
|
||||
Instant instant = Instant.parse(timestamp);
|
||||
return atTime(instant);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a new snapshot at the specified timestamp.
|
||||
*
|
||||
* @param timestamp Instant timestamp
|
||||
* @return This builder for chaining
|
||||
*/
|
||||
public MetricsDataBuilder atTime(Instant timestamp) {
|
||||
if (currentSnapshot != null) {
|
||||
snapshots.add(currentSnapshot);
|
||||
}
|
||||
currentSnapshot = new Snapshot(timestamp);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a counter metric to the current snapshot.
|
||||
*
|
||||
* @param metricName Metric name (will have _total suffix added automatically)
|
||||
* @param value Counter value
|
||||
* @param labels Label key-value pairs (must be even number of strings)
|
||||
* @return This builder for chaining
|
||||
*/
|
||||
public MetricsDataBuilder counter(String metricName, long value, String... labels) {
|
||||
ensureCurrentSnapshot();
|
||||
currentSnapshot.addCounter(metricName, value, parseLabels(labels));
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a timer metric to the current snapshot.
|
||||
*
|
||||
* @param metricName Metric name
|
||||
* @param latencies Array of latency values in milliseconds
|
||||
* @param labels Label key-value pairs (must be even number of strings)
|
||||
* @return This builder for chaining
|
||||
*/
|
||||
public MetricsDataBuilder timer(String metricName, long[] latencies, String... labels) {
|
||||
ensureCurrentSnapshot();
|
||||
currentSnapshot.addTimer(metricName, latencies, parseLabels(labels));
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build and write the database file.
|
||||
*
|
||||
* @throws Exception If database creation fails
|
||||
*/
|
||||
public void build() throws Exception {
|
||||
if (currentSnapshot != null) {
|
||||
snapshots.add(currentSnapshot);
|
||||
}
|
||||
|
||||
if (snapshots.isEmpty()) {
|
||||
throw new IllegalStateException("No snapshots defined. Use atTime() to add snapshots.");
|
||||
}
|
||||
|
||||
NBComponent root = new NBBaseComponent(null, NBLabels.forKV("builder", "test"));
|
||||
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
|
||||
|
||||
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
|
||||
root, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
|
||||
|
||||
for (Snapshot snapshot : snapshots) {
|
||||
snapshot.writeToReporter(reporter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureCurrentSnapshot() {
|
||||
if (currentSnapshot == null) {
|
||||
throw new IllegalStateException("No current snapshot. Call atTime() first.");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> parseLabels(String... labels) {
|
||||
if (labels.length % 2 != 0) {
|
||||
throw new IllegalArgumentException("Labels must be key-value pairs (even number of strings)");
|
||||
}
|
||||
|
||||
Map<String, String> labelMap = new LinkedHashMap<>();
|
||||
for (int i = 0; i < labels.length; i += 2) {
|
||||
labelMap.put(labels[i], labels[i + 1]);
|
||||
}
|
||||
return labelMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a single metrics snapshot at a specific point in time.
|
||||
*/
|
||||
private static class Snapshot {
|
||||
private final Instant timestamp;
|
||||
private final List<NBMetricCounter> counters = new ArrayList<>();
|
||||
private final List<NBMetricTimer> timers = new ArrayList<>();
|
||||
|
||||
Snapshot(Instant timestamp) {
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
void addCounter(String metricName, long value, Map<String, String> labels) {
|
||||
Map<String, String> fullLabels = new LinkedHashMap<>(labels);
|
||||
fullLabels.put("name", metricName);
|
||||
|
||||
NBMetricCounter counter = new NBMetricCounter(
|
||||
NBLabels.forMap(fullLabels),
|
||||
"counter",
|
||||
MetricCategory.Core
|
||||
);
|
||||
counter.inc(value);
|
||||
counters.add(counter);
|
||||
}
|
||||
|
||||
void addTimer(String metricName, long[] latenciesMs, Map<String, String> labels) {
|
||||
Map<String, String> fullLabels = new LinkedHashMap<>(labels);
|
||||
fullLabels.put("name", metricName);
|
||||
|
||||
NBLabels nbLabels = NBLabels.forMap(fullLabels);
|
||||
NBMetricTimer timer = new NBMetricTimer(
|
||||
nbLabels,
|
||||
new DeltaHdrHistogramReservoir(nbLabels, 3),
|
||||
"timer",
|
||||
MetricCategory.Core
|
||||
);
|
||||
|
||||
for (long latencyMs : latenciesMs) {
|
||||
timer.update(Duration.ofMillis(latencyMs));
|
||||
}
|
||||
timers.add(timer);
|
||||
}
|
||||
|
||||
void writeToReporter(SqliteSnapshotReporter reporter) {
|
||||
List<Object> allMetrics = new ArrayList<>();
|
||||
allMetrics.addAll(counters);
|
||||
allMetrics.addAll(timers);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<io.nosqlbench.nb.api.engine.metrics.instruments.NBMetric> metrics =
|
||||
(List<io.nosqlbench.nb.api.engine.metrics.instruments.NBMetric>) (List<?>) allMetrics;
|
||||
|
||||
MetricsView view = MetricsView.capture(metrics, 30000L);
|
||||
|
||||
// Override timestamp by capturing at specific time
|
||||
// Note: MetricsView uses current time, but reporter will use it
|
||||
reporter.onMetricsSnapshot(view);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format an Instant as ISO-8601 for documentation.
|
||||
*/
|
||||
public static String formatTimestamp(Instant instant) {
|
||||
return instant.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT);
|
||||
}
|
||||
}
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.testdata;
|
||||
|
||||
import io.nosqlbench.nb.api.components.core.NBComponent;
|
||||
import io.nosqlbench.nb.api.components.core.NBBaseComponent;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.MetricCategory;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricCounter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricGauge;
|
||||
import io.nosqlbench.nb.api.engine.metrics.instruments.NBMetricTimer;
|
||||
import io.nosqlbench.nb.api.engine.metrics.reporters.SqliteSnapshotReporter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.reporters.MetricInstanceFilter;
|
||||
import io.nosqlbench.nb.api.engine.metrics.DeltaHdrHistogramReservoir;
|
||||
import io.nosqlbench.nb.api.engine.metrics.view.MetricsView;
|
||||
import io.nosqlbench.nb.api.labels.NBLabels;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Generates test databases with various metric scenarios for testing query commands.
|
||||
* Each generator method creates a database with specific characteristics documented
|
||||
* in TEST_DATABASES.md.
|
||||
*/
|
||||
public class TestDatabaseGenerator {
|
||||
|
||||
/**
|
||||
* Generate simple_counter.db
|
||||
*
|
||||
* A basic counter metric with two label dimensions over 5 minutes.
|
||||
*
|
||||
* Schema:
|
||||
* - Metric: activity_ops_total (COUNTER)
|
||||
* - Labels: activity={read,write}, host=server1
|
||||
* - Time range: 5 minutes (10 snapshots @ 30s intervals)
|
||||
* - Values: Linear growth (read: 0→100, write: 0→150)
|
||||
*
|
||||
* Purpose: Test instant queries, simple ranges, basic rate calculations
|
||||
*/
|
||||
public static void generateSimpleCounter(Path outputPath, NBComponent parent) throws Exception {
|
||||
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
|
||||
|
||||
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
|
||||
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
|
||||
|
||||
// Base timestamp: 2025-10-23 10:00:00 UTC
|
||||
long baseTime = 1729681200000L;
|
||||
long intervalMs = 30000L;
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
List<NBMetricCounter> counters = new ArrayList<>();
|
||||
|
||||
// Create read counter
|
||||
NBMetricCounter readCounter = new NBMetricCounter(
|
||||
NBLabels.forKV("name", "activity_ops", "activity", "read", "host", "server1"),
|
||||
"ops",
|
||||
MetricCategory.Core
|
||||
);
|
||||
readCounter.inc(i * 10); // 0, 10, 20, ..., 90
|
||||
|
||||
// Create write counter
|
||||
NBMetricCounter writeCounter = new NBMetricCounter(
|
||||
NBLabels.forKV("name", "activity_ops", "activity", "write", "host", "server1"),
|
||||
"ops",
|
||||
MetricCategory.Core
|
||||
);
|
||||
writeCounter.inc(i * 15); // 0, 15, 30, ..., 135
|
||||
|
||||
counters.add(readCounter);
|
||||
counters.add(writeCounter);
|
||||
|
||||
// Create MetricsView and report snapshot
|
||||
// Note: Timestamps will be auto-generated by the reporter
|
||||
MetricsView view = MetricsView.capture(counters, intervalMs);
|
||||
reporter.onMetricsSnapshot(view);
|
||||
|
||||
// Sleep to create distinct timestamps
|
||||
Thread.sleep(100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate multi_dimensional.db
|
||||
*
|
||||
* Complex metrics with 4 dimensional labels for testing grouping and filtering.
|
||||
*
|
||||
* Schema:
|
||||
* - Metric: request_total (COUNTER)
|
||||
* - Labels: activity={read,write}, host={server1,server2,server3},
|
||||
* region={us-east,us-west}, env={prod,staging}
|
||||
* - Combinations: 2 * 3 * 2 * 2 = 24 unique label sets
|
||||
* - Time range: 2 minutes (4 snapshots @ 30s)
|
||||
*
|
||||
* Purpose: Test label filtering, aggregations with grouping, topk queries
|
||||
*/
|
||||
public static void generateMultiDimensional(Path outputPath, NBComponent parent) throws Exception {
|
||||
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
|
||||
|
||||
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
|
||||
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
|
||||
|
||||
long baseTime = 1729681200000L;
|
||||
long intervalMs = 30000L;
|
||||
String[] activities = {"read", "write"};
|
||||
String[] hosts = {"server1", "server2", "server3"};
|
||||
String[] regions = {"us-east", "us-west"};
|
||||
String[] envs = {"prod", "staging"};
|
||||
|
||||
for (int snapshot = 0; snapshot < 4; snapshot++) {
|
||||
List<NBMetricCounter> counters = new ArrayList<>();
|
||||
int counter = 0;
|
||||
|
||||
for (String activity : activities) {
|
||||
for (String host : hosts) {
|
||||
for (String region : regions) {
|
||||
for (String env : envs) {
|
||||
NBMetricCounter metric = new NBMetricCounter(
|
||||
NBLabels.forKV(
|
||||
"name", "requests",
|
||||
"activity", activity,
|
||||
"host", host,
|
||||
"region", region,
|
||||
"env", env
|
||||
),
|
||||
"requests",
|
||||
MetricCategory.Core
|
||||
);
|
||||
|
||||
// Value varies by combination and snapshot
|
||||
long value = (counter * 10) + (snapshot * 100);
|
||||
metric.inc(value);
|
||||
counters.add(metric);
|
||||
counter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MetricsView view = MetricsView.capture(counters, intervalMs);
|
||||
reporter.onMetricsSnapshot(view);
|
||||
Thread.sleep(100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate latency_timers.db
|
||||
*
|
||||
* Summary/timer metrics with quantiles for latency analysis.
|
||||
*
|
||||
* Schema:
|
||||
* - Metric: operation_latency_ms (TIMER/SUMMARY)
|
||||
* - Labels: operation={select,insert,update,delete}
|
||||
* - Quantiles: p50, p75, p90, p95, p98, p99, p999
|
||||
* - Statistics: count, sum, min, max, mean, stddev
|
||||
* - Time range: 3 minutes (6 snapshots @ 30s)
|
||||
*
|
||||
* Purpose: Test quantile queries, statistical aggregations
|
||||
*/
|
||||
public static void generateLatencyTimers(Path outputPath, NBComponent parent) throws Exception {
|
||||
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
|
||||
|
||||
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
|
||||
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
|
||||
|
||||
long baseTime = 1729681200000L;
|
||||
long intervalMs = 30000L;
|
||||
String[] operations = {"select", "insert", "update", "delete"};
|
||||
|
||||
for (int snapshot = 0; snapshot < 6; snapshot++) {
|
||||
List<NBMetricTimer> timers = new ArrayList<>();
|
||||
|
||||
for (int opIdx = 0; opIdx < operations.length; opIdx++) {
|
||||
String operation = operations[opIdx];
|
||||
NBLabels labels = NBLabels.forKV(
|
||||
"name", "operation_latency",
|
||||
"operation", operation
|
||||
);
|
||||
|
||||
NBMetricTimer timer = new NBMetricTimer(
|
||||
labels,
|
||||
new DeltaHdrHistogramReservoir(labels, 3),
|
||||
"latency",
|
||||
MetricCategory.Core
|
||||
);
|
||||
|
||||
// Vary latencies by operation type and over time
|
||||
long baseLatency = (opIdx + 1) * 10_000_000L; // 10ms, 20ms, 30ms, 40ms in nanoseconds
|
||||
long drift = snapshot * 2_000_000L; // 2ms increase per snapshot
|
||||
|
||||
// Record latencies with some variation
|
||||
int recordCount = 100 + (snapshot * 20);
|
||||
for (int i = 0; i < recordCount; i++) {
|
||||
// Create a distribution of latencies
|
||||
long latency;
|
||||
if (i < recordCount * 0.5) {
|
||||
latency = baseLatency + drift; // p50
|
||||
} else if (i < recordCount * 0.75) {
|
||||
latency = (long) ((baseLatency + drift) * 1.5); // p75
|
||||
} else if (i < recordCount * 0.95) {
|
||||
latency = (long) ((baseLatency + drift) * 2.0); // p95
|
||||
} else {
|
||||
latency = (long) ((baseLatency + drift) * 3.0); // p99
|
||||
}
|
||||
timer.update(Duration.ofNanos(latency));
|
||||
}
|
||||
|
||||
timers.add(timer);
|
||||
}
|
||||
|
||||
long timestamp = baseTime + (snapshot * intervalMs);
|
||||
MetricsView view = MetricsView.capture(timers, intervalMs);
|
||||
reporter.onMetricsSnapshot(view);
|
||||
Thread.sleep(100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate rate_calculations.db
|
||||
*
|
||||
* Counter metrics with predictable growth patterns for testing rate() and increase().
|
||||
*
|
||||
* Schema:
|
||||
* - Metrics: counter_total (COUNTER)
|
||||
* - Labels: pattern={linear,exponential,step}
|
||||
* - Time range: 1 hour (120 snapshots @ 30s)
|
||||
* - Patterns:
|
||||
* - linear: grows by 10 per snapshot (constant rate)
|
||||
* - exponential: grows by 1.01x per snapshot
|
||||
* - step: 100 for 10 snapshots, then jump to 200 for 10, etc.
|
||||
*
|
||||
* Purpose: Test rate() and increase() with known expected results
|
||||
*/
|
||||
public static void generateRateCalculations(Path outputPath, NBComponent parent) throws Exception {
|
||||
String jdbcUrl = "jdbc:sqlite:" + outputPath.toAbsolutePath();
|
||||
|
||||
try (SqliteSnapshotReporter reporter = new SqliteSnapshotReporter(
|
||||
parent, jdbcUrl, 30000L, new MetricInstanceFilter(), NBLabels.forKV())) {
|
||||
|
||||
long baseTime = 1729681200000L;
|
||||
long intervalMs = 30000L;
|
||||
double exponentialValue = 100.0;
|
||||
|
||||
for (int i = 0; i < 120; i++) {
|
||||
List<NBMetricCounter> counters = new ArrayList<>();
|
||||
|
||||
// Linear counter: constant rate of 10 per 30s = 0.333 ops/sec
|
||||
NBMetricCounter linearCounter = new NBMetricCounter(
|
||||
NBLabels.forKV("name", "patterns", "pattern", "linear"),
|
||||
"counter",
|
||||
MetricCategory.Core
|
||||
);
|
||||
linearCounter.inc(i * 10);
|
||||
counters.add(linearCounter);
|
||||
|
||||
// Exponential counter: exponential growth
|
||||
exponentialValue *= 1.01; // 1% growth per snapshot
|
||||
NBMetricCounter expCounter = new NBMetricCounter(
|
||||
NBLabels.forKV("name", "patterns", "pattern", "exponential"),
|
||||
"counter",
|
||||
MetricCategory.Core
|
||||
);
|
||||
expCounter.inc((long) exponentialValue);
|
||||
counters.add(expCounter);
|
||||
|
||||
// Step counter: steps every 10 snapshots
|
||||
long stepValue = ((i / 10) + 1) * 100;
|
||||
NBMetricCounter stepCounter = new NBMetricCounter(
|
||||
NBLabels.forKV("name", "patterns", "pattern", "step"),
|
||||
"counter",
|
||||
MetricCategory.Core
|
||||
);
|
||||
stepCounter.inc(stepValue);
|
||||
counters.add(stepCounter);
|
||||
|
||||
MetricsView view = MetricsView.capture(counters, intervalMs);
|
||||
reporter.onMetricsSnapshot(view);
|
||||
Thread.sleep(50); // Shorter sleep for many snapshots
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main method to generate all test databases.
|
||||
* Run this to regenerate test data after schema changes.
|
||||
*/
|
||||
public static void main(String[] args) throws Exception {
|
||||
Path testDataDir = Path.of("nb-apis/nb-mql-api/src/test/resources/testdata");
|
||||
|
||||
// Create a minimal NBComponent for the reporter
|
||||
NBComponent root = new NBBaseComponent(null, NBLabels.forKV("generator", "testdata"));
|
||||
|
||||
System.out.println("Generating test databases...");
|
||||
|
||||
generateSimpleCounter(testDataDir.resolve("simple_counter.db"), root);
|
||||
System.out.println("✓ Generated simple_counter.db");
|
||||
|
||||
generateMultiDimensional(testDataDir.resolve("multi_dimensional.db"), root);
|
||||
System.out.println("✓ Generated multi_dimensional.db");
|
||||
|
||||
generateLatencyTimers(testDataDir.resolve("latency_timers.db"), root);
|
||||
System.out.println("✓ Generated latency_timers.db");
|
||||
|
||||
generateRateCalculations(testDataDir.resolve("rate_calculations.db"), root);
|
||||
System.out.println("✓ Generated rate_calculations.db");
|
||||
|
||||
System.out.println("\nAll test databases generated successfully!");
|
||||
System.out.println("Location: " + testDataDir.toAbsolutePath());
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
/*
|
||||
* Copyright (c) nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.mql.testdata;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/**
|
||||
* Utility for loading test databases from src/test/resources/testdata/.
|
||||
* Provides easy access to pre-generated test databases for unit and integration tests.
|
||||
*/
|
||||
public class TestDatabaseLoader {
|
||||
|
||||
private static final String TEST_DATA_DIR = "testdata";
|
||||
|
||||
/**
|
||||
* Available test databases with their purposes.
|
||||
*/
|
||||
public enum TestDatabase {
|
||||
SIMPLE_COUNTER("simple_counter.db", "Basic counters for instant/rate queries"),
|
||||
MULTI_DIMENSIONAL("multi_dimensional.db", "Complex 4D labels for aggregations"),
|
||||
LATENCY_TIMERS("latency_timers.db", "Timer metrics with quantiles"),
|
||||
RATE_CALCULATIONS("rate_calculations.db", "Predictable growth patterns for rate testing");
|
||||
|
||||
private final String filename;
|
||||
private final String description;
|
||||
|
||||
TestDatabase(String filename, String description) {
|
||||
this.filename = filename;
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to a test database by enum.
|
||||
*
|
||||
* @param database The test database enum
|
||||
* @return Path to the database file
|
||||
* @throws IOException If the database file cannot be found
|
||||
*/
|
||||
public static Path getDatabase(TestDatabase database) throws IOException {
|
||||
return getDatabase(database.getFilename());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to a test database by filename.
|
||||
*
|
||||
* @param filename Name of the database file (e.g., "simple_counter.db")
|
||||
* @return Path to the database file
|
||||
* @throws IOException If the database file cannot be found
|
||||
*/
|
||||
public static Path getDatabase(String filename) throws IOException {
|
||||
String resourcePath = TEST_DATA_DIR + "/" + filename;
|
||||
URL resource = TestDatabaseLoader.class.getClassLoader().getResource(resourcePath);
|
||||
|
||||
if (resource == null) {
|
||||
throw new IOException(
|
||||
"Test database not found: " + resourcePath + "\n" +
|
||||
"Make sure test databases have been generated by running TestDatabaseGenerator.main()"
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return Paths.get(resource.toURI());
|
||||
} catch (URISyntaxException e) {
|
||||
throw new IOException("Invalid URI for test database: " + resourcePath, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a test database exists.
|
||||
*
|
||||
* @param database The test database enum
|
||||
* @return true if the database file exists
|
||||
*/
|
||||
public static boolean exists(TestDatabase database) {
|
||||
try {
|
||||
getDatabase(database);
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a descriptive message about available test databases.
|
||||
*
|
||||
* @return Multi-line string describing all test databases
|
||||
*/
|
||||
public static String getAvailableDatabases() {
|
||||
StringBuilder sb = new StringBuilder("Available test databases:\n");
|
||||
for (TestDatabase db : TestDatabase.values()) {
|
||||
sb.append(String.format(" - %s: %s\n",
|
||||
db.getFilename(),
|
||||
db.getDescription()));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private TestDatabaseLoader() {
|
||||
// Utility class - prevent instantiation
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
# Test Database Documentation
|
||||
|
||||
This directory contains pre-generated SQLite metrics databases for testing the MetricsQL query engine. Each database represents a specific scenario with known data patterns.
|
||||
|
||||
## Overview
|
||||
|
||||
All databases use the NoSQLBench OpenMetrics-aligned schema created by `SqliteSnapshotReporter`. They contain realistic metric data generated using actual metric instruments (`NBMetricCounter`, `NBMetricTimer`).
|
||||
|
||||
---
|
||||
|
||||
## simple_counter.db
|
||||
|
||||
### Purpose
|
||||
Test basic counter queries, instant values, simple rate calculations, and time-series ranges.
|
||||
|
||||
### Schema
|
||||
- **Metric Family**: `activity_ops`
|
||||
- **Sample**: `activity_ops_total` (COUNTER)
|
||||
- **Type**: COUNTER
|
||||
|
||||
### Label Dimensions
|
||||
- `activity`: {read, write}
|
||||
- `host`: {server1}
|
||||
|
||||
### Data Characteristics
|
||||
- **Snapshots**: 10 (taken ~100ms apart during generation)
|
||||
- **Label Combinations**: 2 (read, write)
|
||||
- **Total Samples**: 20
|
||||
- **Growth Pattern**: Linear increments
|
||||
- `activity=read`: increments by 10 per snapshot (0, 10, 20, ..., 90)
|
||||
- `activity=write`: increments by 15 per snapshot (0, 15, 30, ..., 135)
|
||||
|
||||
### Expected Query Results
|
||||
|
||||
**Instant Query (latest value):**
|
||||
```sql
|
||||
SELECT sv.value, lv.value as activity
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
|
||||
JOIN label_key lk ON lsm.label_key_id = lk.id
|
||||
JOIN label_value lv ON lsm.label_value_id = lv.id
|
||||
WHERE sn.sample = 'activity_ops_total'
|
||||
AND lk.name = 'activity'
|
||||
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
ORDER BY activity;
|
||||
```
|
||||
Expected: `{activity=read: 90, activity=write: 135}`
|
||||
|
||||
**Count Total Samples:**
|
||||
```sql
|
||||
SELECT COUNT(*) FROM sample_value;
|
||||
```
|
||||
Expected: `20`
|
||||
|
||||
**Rate Calculation (approximate):**
|
||||
For `activity=read`, the rate should be approximately **10 ops per 100ms** = **100 ops/sec** (simplified; actual timing varies slightly)
|
||||
|
||||
### Use Cases
|
||||
- Testing instant value queries
|
||||
- Basic label filtering
|
||||
- Simple rate calculations
|
||||
- Time-series data retrieval
|
||||
|
||||
---
|
||||
|
||||
## multi_dimensional.db
|
||||
|
||||
### Purpose
|
||||
Test complex label filtering, multi-dimensional aggregations, grouping operations, and topk queries.
|
||||
|
||||
### Schema
|
||||
- **Metric Family**: `requests`
|
||||
- **Sample**: `request_total` (COUNTER)
|
||||
- **Type**: COUNTER
|
||||
|
||||
### Label Dimensions
|
||||
- `activity`: {read, write}
|
||||
- `host`: {server1, server2, server3}
|
||||
- `region`: {us-east, us-west}
|
||||
- `env`: {prod, staging}
|
||||
|
||||
### Data Characteristics
|
||||
- **Snapshots**: 4
|
||||
- **Label Combinations**: 24 (2 × 3 × 2 × 2)
|
||||
- **Total Samples**: 96
|
||||
- **Value Pattern**: `(combo_index * 10) + (snapshot_index * 100)`
|
||||
- First snapshot: values 0, 10, 20, ..., 230
|
||||
- Second snapshot: values 100, 110, 120, ..., 330
|
||||
- And so on...
|
||||
|
||||
### Expected Query Results
|
||||
|
||||
**Count by Environment:**
|
||||
```sql
|
||||
SELECT lv.value as env, COUNT(*) as sample_count
|
||||
FROM sample_value sv
|
||||
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
|
||||
JOIN label_key lk ON lsm.label_key_id = lk.id
|
||||
JOIN label_value lv ON lsm.label_value_id = lv.id
|
||||
WHERE lk.name = 'env'
|
||||
GROUP BY env;
|
||||
```
|
||||
Expected: `{prod: 48, staging: 48}`
|
||||
|
||||
**Unique Label Sets:**
|
||||
```sql
|
||||
SELECT COUNT(DISTINCT label_set_id) FROM sample_value;
|
||||
```
|
||||
Expected: `24`
|
||||
|
||||
### Use Cases
|
||||
- Multi-dimensional filtering (e.g., `activity=write AND region=us-east AND env=prod`)
|
||||
- Aggregations with grouping (e.g., `sum by (region, env)`)
|
||||
- Top-K queries (e.g., "top 5 hosts by request count")
|
||||
- Complex label pattern matching
|
||||
|
||||
---
|
||||
|
||||
## latency_timers.db
|
||||
|
||||
### Purpose
|
||||
Test summary/timer metrics with quantiles, statistical aggregations, and rate statistics.
|
||||
|
||||
### Schema
|
||||
- **Metric Family**: `operation_latency`
|
||||
- **Sample**: `operation_latency_ms` (TIMER/SUMMARY)
|
||||
- **Type**: TIMER
|
||||
|
||||
### Label Dimensions
|
||||
- `operation`: {select, insert, update, delete}
|
||||
|
||||
### Data Characteristics
|
||||
- **Snapshots**: 6
|
||||
- **Operations**: 4
|
||||
- **Total Sample Values**: 24
|
||||
- **Quantiles**: p50, p75, p90, p95, p98, p99, p999 (stored in `sample_quantile` table)
|
||||
- **Statistics**: count, sum, min, max, mean, stddev (in `sample_value`)
|
||||
- **Rate Statistics**: mean, m1, m5, m15 (in `sample_rate` table)
|
||||
|
||||
### Latency Patterns
|
||||
Base latencies by operation (in nanoseconds):
|
||||
- `select`: 10ms base, increases by 2ms per snapshot
|
||||
- `insert`: 20ms base, increases by 2ms per snapshot
|
||||
- `update`: 30ms base, increases by 2ms per snapshot
|
||||
- `delete`: 40ms base, increases by 2ms per snapshot
|
||||
|
||||
Distribution within each snapshot:
|
||||
- 50% of samples at base latency (p50)
|
||||
- 25% at 1.5× base (p75)
|
||||
- 20% at 2.0× base (p95)
|
||||
- 5% at 3.0× base (p99)
|
||||
|
||||
### Expected Query Results
|
||||
|
||||
**Get p95 latency for select operations:**
|
||||
```sql
|
||||
SELECT sq.quantile_value
|
||||
FROM sample_quantile sq
|
||||
JOIN sample_value sv ON sq.sample_value_id = sv.id
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
|
||||
JOIN label_key lk ON lsm.label_key_id = lk.id
|
||||
JOIN label_value lv ON lsm.label_value_id = lv.id
|
||||
WHERE sn.sample = 'operation_latency_ms'
|
||||
AND lk.name = 'operation'
|
||||
AND lv.value = 'select'
|
||||
AND sq.quantile = 0.95
|
||||
AND sv.timestamp_ms = (SELECT MAX(timestamp_ms) FROM sample_value)
|
||||
LIMIT 1;
|
||||
```
|
||||
Expected: Approximately `22,000,000` nanoseconds (22ms) for latest snapshot
|
||||
|
||||
**Count quantile entries:**
|
||||
```sql
|
||||
SELECT COUNT(*) FROM sample_quantile;
|
||||
```
|
||||
Expected: `168` (24 samples × 7 quantiles each)
|
||||
|
||||
### Use Cases
|
||||
- Quantile extraction (p50, p95, p99)
|
||||
- Statistical aggregations (mean, stddev, min, max)
|
||||
- Latency trend analysis over time
|
||||
- Rate statistics queries (m1, m5, m15)
|
||||
|
||||
---
|
||||
|
||||
## rate_calculations.db
|
||||
|
||||
### Purpose
|
||||
Test rate() and increase() commands with predictable, mathematically known patterns over extended time periods.
|
||||
|
||||
### Schema
|
||||
- **Metric Family**: `patterns`
|
||||
- **Sample**: `counter_total` (COUNTER)
|
||||
- **Type**: COUNTER
|
||||
|
||||
### Label Dimensions
|
||||
- `pattern`: {linear, exponential, step}
|
||||
|
||||
### Data Characteristics
|
||||
- **Snapshots**: 120 (simulating 1 hour at 30s intervals)
|
||||
- **Patterns**: 3
|
||||
- **Total Samples**: 360
|
||||
- **Interval Between Snapshots**: ~50ms (during generation)
|
||||
|
||||
### Growth Patterns
|
||||
|
||||
#### Linear Pattern (`pattern=linear`)
|
||||
- Growth: **+10 per snapshot**
|
||||
- Values: 0, 10, 20, 30, ..., 1190
|
||||
- Expected rate: **10 ops / 50ms** = **200 ops/sec** (constant)
|
||||
- Total increase over all snapshots: **1190**
|
||||
|
||||
#### Exponential Pattern (`pattern=exponential`)
|
||||
- Growth: **×1.01 per snapshot** (1% compound growth)
|
||||
- Starting value: 100
|
||||
- Values: 100, 101, 102.01, 103.03, ...
|
||||
- Expected rate: Increases over time
|
||||
- Formula: `value[i] = 100 × (1.01)^i`
|
||||
|
||||
#### Step Pattern (`pattern=step`)
|
||||
- Growth: **Steps every 10 snapshots**
|
||||
- Values: 100 (snapshots 0-9), 200 (10-19), 300 (20-29), ..., 1200 (110-119)
|
||||
- Expected rate: **0 within step, jump at boundaries**
|
||||
- Total steps: 12
|
||||
|
||||
### Expected Query Results
|
||||
|
||||
**Linear pattern increase over all snapshots:**
|
||||
```sql
|
||||
SELECT MAX(sv.value) - MIN(sv.value) as total_increase
|
||||
FROM sample_value sv
|
||||
JOIN sample_name sn ON sv.sample_name_id = sn.id
|
||||
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
|
||||
JOIN label_key lk ON lsm.label_key_id = lk.id
|
||||
JOIN label_value lv ON lsm.label_value_id = lv.id
|
||||
WHERE sn.sample = 'counter_total'
|
||||
AND lk.name = 'pattern'
|
||||
AND lv.value = 'linear';
|
||||
```
|
||||
Expected: `1190`
|
||||
|
||||
**Count snapshots per pattern:**
|
||||
```sql
|
||||
SELECT lv.value as pattern, COUNT(*) as snapshots
|
||||
FROM sample_value sv
|
||||
JOIN label_set_membership lsm ON sv.label_set_id = lsm.label_set_id
|
||||
JOIN label_key lk ON lsm.label_key_id = lk.id
|
||||
JOIN label_value lv ON lsm.label_value_id = lv.id
|
||||
WHERE lk.name = 'pattern'
|
||||
GROUP BY pattern;
|
||||
```
|
||||
Expected: `{linear: 120, exponential: 120, step: 120}`
|
||||
|
||||
### Use Cases
|
||||
- Testing rate() with constant rate (linear)
|
||||
- Testing rate() with changing rate (exponential)
|
||||
- Testing rate() with discontinuous data (step)
|
||||
- Testing increase() over various time windows
|
||||
- Validating rate calculation algorithms with known results
|
||||
|
||||
---
|
||||
|
||||
## Schema Reference
|
||||
|
||||
All databases share this schema (created by `SqliteSnapshotReporter`):
|
||||
|
||||
```
|
||||
metric_family (id, name, help, unit, type)
|
||||
└─ sample_name (id, metric_family_id, sample)
|
||||
└─ sample_value (id, sample_name_id, label_set_id, timestamp_ms, value, count, sum, min, max, mean, stddev)
|
||||
├─ sample_quantile (sample_value_id, quantile, quantile_value)
|
||||
├─ sample_rate (sample_value_id, rate_type, rate_value)
|
||||
└─ sample_histogram (sample_value_id, start_seconds, interval_seconds, max_value, histogram_base64)
|
||||
|
||||
label_key (id, name)
|
||||
label_value (id, value)
|
||||
label_set (id, hash)
|
||||
label_set_membership (label_set_id, label_key_id, label_value_id)
|
||||
```
|
||||
|
||||
## Regenerating Test Databases
|
||||
|
||||
To regenerate all test databases (e.g., after schema changes):
|
||||
|
||||
```bash
|
||||
mvn exec:java -pl nb-apis/nb-mql-api \
|
||||
-Dexec.mainClass="io.nosqlbench.nb.mql.testdata.TestDatabaseGenerator" \
|
||||
-Dexec.classpathScope=test
|
||||
```
|
||||
|
||||
The generator will overwrite existing databases in `src/test/resources/testdata/`.
|
||||
|
||||
## Using in Tests
|
||||
|
||||
Use `TestDatabaseLoader` utility to access these databases in your tests:
|
||||
|
||||
```java
|
||||
import io.nosqlbench.nb.mql.testdata.TestDatabaseLoader;
|
||||
|
||||
Path dbPath = TestDatabaseLoader.getDatabase("simple_counter.db");
|
||||
Connection conn = MetricsDatabaseReader.connect(dbPath);
|
||||
// ... run queries
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
*Generated: 2025-10-23*
|
||||
*Schema Version: OpenMetrics-aligned (SqliteSnapshotReporter)*
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -35,6 +35,7 @@
|
||||
<module>adapters-api</module>
|
||||
<module>expr-api</module>
|
||||
<module>nb-api</module>
|
||||
<module>nb-mql-api</module>
|
||||
<module>nb-apis-all</module>
|
||||
</modules>
|
||||
|
||||
|
||||
@@ -55,6 +55,12 @@
|
||||
<artifactId>nb-engine-extensions</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.nosqlbench</groupId>
|
||||
<artifactId>nb-mql-api</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<!-- <plugins>-->
|
||||
|
||||
@@ -182,6 +182,11 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
|
||||
.activate();
|
||||
ConfigurationFactory.setConfigurationFactory(NBCLI.loggerConfig); // THIS should be the first time log4j2 is invoked!
|
||||
|
||||
// Add shutdown handler to timestamp symlinks when session ends
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
||||
NBCLI.loggerConfig.timestampSymlinksOnShutdown();
|
||||
}, "symlink-timestamp-shutdown"));
|
||||
|
||||
NBCLI.logger = LogManager.getLogger("NBCLI"); // TODO: Detect if the logger config was already initialized (error)
|
||||
NBCLI.loggerConfig.purgeOldFiles(LogManager.getLogger("SCENARIO"));
|
||||
if (NBCLI.logger.isInfoEnabled())
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Metrics Reporting Guide
|
||||
contin# Metrics Reporting Guide
|
||||
|
||||
## What You Can Configure
|
||||
|
||||
|
||||
+38
@@ -485,4 +485,42 @@ public class NBLoggerConfig extends ConfigurationFactory {
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Update modification time on symlinks to reflect shutdown time.
|
||||
* Called by shutdown hook to touch symlinks so their mtime shows when the session ended.
|
||||
* Updates symlinks for: scenario.log, status.log, runtime.log, verify.log, metrics.db, heartbeat.log
|
||||
*/
|
||||
public void timestampSymlinksOnShutdown() {
|
||||
if (loggerDir == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// List of symlinks to update
|
||||
String[] symlinks = {
|
||||
"scenario.log",
|
||||
"status.log",
|
||||
"runtime.log",
|
||||
"verify.log",
|
||||
"metrics.db",
|
||||
"heartbeat.log"
|
||||
};
|
||||
|
||||
for (String linkName : symlinks) {
|
||||
try {
|
||||
Path linkPath = loggerDir.resolve(linkName);
|
||||
|
||||
if (!Files.exists(linkPath, java.nio.file.LinkOption.NOFOLLOW_LINKS)) {
|
||||
continue; // Symlink doesn't exist, skip
|
||||
}
|
||||
|
||||
// Update the modification time of the symlink itself (not the target file)
|
||||
Files.setLastModifiedTime(linkPath,
|
||||
java.nio.file.attribute.FileTime.from(java.time.Instant.now()));
|
||||
|
||||
} catch (IOException | UnsupportedOperationException e) {
|
||||
// Silently ignore - symlinks may not be supported or may have issues during shutdown
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user