diff --git a/README.md b/README.md index dc79a6c55..fd4512032 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ that are not found in any other tool. - You can generate virtual data sets of arbitrary size, with deterministic data and statistically shaped values. - You can design custom workloads that emulate your application, contained in a single file, based on statement templates - no IDE or coding required. -- You can immediately plot your results in a docker and grafana stack on Linux with a single command line option. - When needed, you can open the access panels and rewire the runtime behavior of NoSQLBench to do advanced testing, including a full scripting environment with Javascript. @@ -84,7 +83,7 @@ available, but more work is needed to support them fully. Here is what is suppor 1. on Linux, all features are supported, for both `nb5.jar` as well as the appimage binary `nb` 2. on Mac, all features are supported, with `nb5.jar`. -3. On Windows, with `nb5.jar` all features are supported, except `--docker-metrics`. +3. on Windows, all features are supported, with `nb5.jar`. ## Thanks @@ -111,4 +110,4 @@ available, but more work is needed to support them fully. Here is what is suppor ## Contributors Checkout all our wonderful contributors [here](./CONTRIBUTING.md#contributors). ---- \ No newline at end of file +--- diff --git a/adapter-kafka/src/main/resources/start_kafka_consumer.sh b/adapter-kafka/src/main/resources/start_kafka_consumer.sh index ee9eee05a..0994e68bd 100755 --- a/adapter-kafka/src/main/resources/start_kafka_consumer.sh +++ b/adapter-kafka/src/main/resources/start_kafka_consumer.sh @@ -26,7 +26,6 @@ java -jar nb5/target/nb5.jar \ driver=kafka \ -vv \ --report-interval 5 \ - --docker-metrics \ cycles=${CYCLES} \ threads=1 \ num_clnt=1 \ diff --git a/adapter-kafka/src/main/resources/start_kafka_producer.sh b/adapter-kafka/src/main/resources/start_kafka_producer.sh index 2e3fbe26a..96999cc28 100755 --- a/adapter-kafka/src/main/resources/start_kafka_producer.sh +++ b/adapter-kafka/src/main/resources/start_kafka_producer.sh @@ -27,7 +27,6 @@ while [[ 1 -eq 1 ]]; do driver=kafka \ -vv \ --report-interval 5 \ - --docker-metrics \ cycles="${CYCLES}" \ threads=1 \ num_clnt=1 \ diff --git a/adapter-pulsar/src/main/resources/start_pulsar_consumer.sh b/adapter-pulsar/src/main/resources/start_pulsar_consumer.sh index 67f98b23a..2b6fa7288 100755 --- a/adapter-pulsar/src/main/resources/start_pulsar_consumer.sh +++ b/adapter-pulsar/src/main/resources/start_pulsar_consumer.sh @@ -26,7 +26,6 @@ java -jar nb5/target/nb5.jar \ driver=pulsar \ -vv \ --report-interval 5 \ - --docker-metrics \ cycles=${CYCLES} \ yaml="${SCRIPT_DIR}/scenarios/consumer_4KB_workload.yaml" \ config="${SCRIPT_DIR}/conf/pulsar_config.properties" diff --git a/adapter-pulsar/src/main/resources/start_pulsar_producer.sh b/adapter-pulsar/src/main/resources/start_pulsar_producer.sh index d1c980af1..1ddd4ee29 100755 --- a/adapter-pulsar/src/main/resources/start_pulsar_producer.sh +++ b/adapter-pulsar/src/main/resources/start_pulsar_producer.sh @@ -28,7 +28,6 @@ while [[ 1 -eq 1 ]]; do driver=pulsar \ -vv \ --report-interval 5 \ - --docker-metrics \ cycles="${CYCLES}" \ cyclerate="${CYCLERATE}" \ threads=1 \ diff --git a/devdocs/sketches/annotations.md b/devdocs/sketches/annotations.md index ba6bb95b0..ce64d3651 100644 --- a/devdocs/sketches/annotations.md +++ b/devdocs/sketches/annotations.md @@ -114,10 +114,6 @@ anywhere details would be. ## Annotation Implementations -NoSQLBench comes with two built-in annotation implementations, -which are configured automatically. By default, the logging -annotator is enabled. +NoSQLBench comes with a built-in annotation implementations, the logging annotator, +which is configured automatically. -If --docker-metrics is used, then the grafana annotator is enabled. -In this case, if needed, default admin credentials will be used -to create an API key, which will be cached on the local filesystem. diff --git a/engine-cli/pom.xml b/engine-cli/pom.xml index 7aa5025c4..afcc3d4af 100644 --- a/engine-cli/pom.xml +++ b/engine-cli/pom.xml @@ -42,11 +42,11 @@ ${revision} - - io.nosqlbench - engine-docker - ${revision} - + + + + + diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java index 4331a47e4..9d637bc26 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java @@ -49,7 +49,6 @@ import io.nosqlbench.engine.core.lifecycle.scenario.script.MetricsMapper; import io.nosqlbench.engine.core.lifecycle.scenario.Scenario; import io.nosqlbench.engine.core.lifecycle.scenario.ScenariosExecutor; import io.nosqlbench.engine.core.lifecycle.scenario.script.ScriptParams; -import io.nosqlbench.engine.docker.DockerMetricsManager; import io.nosqlbench.nb.annotations.Maturity; import io.nosqlbench.nb.annotations.Service; import io.nosqlbench.nb.annotations.ServiceSelector; @@ -210,51 +209,13 @@ public class NBCLI implements Function, NBLabeledElement { } } - - final boolean dockerMetrics = globalOptions.wantsDockerMetrics(); - final String dockerMetricsAt = globalOptions.wantsDockerMetricsAt(); String reportGraphiteTo = globalOptions.wantsReportGraphiteTo(); String annotatorsConfig = globalOptions.getAnnotatorsConfig(); String promPushConfig = globalOptions.getPromPushConfig(); final String reportPromPushTo = globalOptions.wantsReportPromPushTo(); - - final int mOpts = (dockerMetrics ? 1 : 0) - + ((null != dockerMetricsAt) ? 1 : 0) - + ((null != reportGraphiteTo) ? 1 : 0); - - if ((1 < mOpts) && ((null == reportGraphiteTo) || (null == annotatorsConfig))) - throw new BasicError("You have multiple conflicting options which attempt to set\n" + - " the destination for metrics and annotations. Please select only one of\n" + - " --docker-metrics, --docker-metrics-at , or other options like \n" + - " --report-graphite-to and --annotators \n" + - " For more details, see run 'nb help docker-metrics'"); - String graphiteMetricsAddress = null; - if (dockerMetrics) { - // Setup docker stack for local docker metrics - NBCLI.logger.info("Docker metrics is enabled. Docker must be installed for this to work"); - final DockerMetricsManager dmh = new DockerMetricsManager(); - final Map dashboardOptions = Map.of( - DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag(), - DockerMetricsManager.PROM_TAG, globalOptions.getDockerPromTag(), - DockerMetricsManager.TSDB_RETENTION, String.valueOf(globalOptions.getDockerPromRetentionDays()), - DockerMetricsManager.GRAPHITE_SAMPLE_EXPIRY, "10m", - DockerMetricsManager.GRAPHITE_CACHE_SIZE, "5000", - DockerMetricsManager.GRAPHITE_LOG_LEVEL, globalOptions.getGraphiteLogLevel(), - DockerMetricsManager.GRAPHITE_LOG_FORMAT, "logfmt" - - ); - dmh.startMetrics(dashboardOptions); - final String warn = "Docker Containers are started, for grafana and prometheus, hit" + - " these urls in your browser: http://:3000 and http://:9090"; - NBCLI.logger.warn(warn); - graphiteMetricsAddress = "localhost"; - } else if (null != dockerMetricsAt) { - graphiteMetricsAddress = dockerMetricsAt; - } - if (annotatorsConfig == null || annotatorsConfig.isBlank()) { List> annotatorsConfigs = new ArrayList<>(); annotatorsConfigs.add(Map.of( diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java index 2851c85ad..3f61deab3 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java @@ -127,12 +127,6 @@ public class NBCLIOptions { private static final String LOG_LEVEL_OVERRIDE = "--log-level-override"; private static final String ENABLE_CHART = "--enable-chart"; - private static final String DOCKER_METRICS = "--docker-metrics"; - private static final String DOCKER_METRICS_AT = "--docker-metrics-at"; - private static final String DOCKER_GRAFANA_TAG = "--docker-grafana-tag"; - private static final String DOCKER_PROM_TAG = "--docker-prom-tag"; - private static final String DOCKER_PROM_RETENTION_DAYS = "--docker-prom-retention-days"; - private static final String GRAALJS_ENGINE = "--graaljs"; private static final String DEFAULT_CONSOLE_PATTERN = "TERSE"; @@ -176,7 +170,6 @@ public class NBCLIOptions { private NBLogLevel logsLevel = NBLogLevel.INFO; private Map logLevelsOverrides = new HashMap<>(); private boolean enableChart; - private boolean dockerMetrics; private boolean wantsListScenarios; private boolean wantsListScripts; private String wantsToCopyWorkload; @@ -184,19 +177,15 @@ public class NBCLIOptions { private final List wantsToIncludePaths = new ArrayList<>(); private Engine engine = Engine.Graalvm; private int hdr_digits = 3; - private String docker_grafana_tag = "7.3.4"; - private String docker_prom_tag = "latest"; private boolean showStackTraces; private boolean compileScript; private String scriptFile; private String[] annotateEvents = {"ALL"}; - private String dockerMetricsHost; private String annotatorsConfig = ""; private String promPushConfig = ""; private String statedirs = NBStatePath.NB_STATEDIR_PATHS; private Path statepath; private final String hdrForChartFileName = NBCLIOptions.DEFAULT_CHART_HDR_LOG_NAME; - private String dockerPromRetentionDays = "3650d"; private String reportSummaryTo = NBCLIOptions.REPORT_SUMMARY_TO_DEFAULT; private boolean enableAnsi = (null != System.getenv("TERM")) && !System.getenv("TERM").isEmpty(); private Maturity minMaturity = Maturity.Unspecified; @@ -236,10 +225,6 @@ public class NBCLIOptions { return this.hdrForChartFileName; } - public String getDockerPromRetentionDays() { - return dockerPromRetentionDays; - } - public String getReportSummaryTo() { return this.reportSummaryTo; } @@ -420,18 +405,6 @@ public class NBCLIOptions { arglist.removeFirst(); this.workspacesDirectory = this.readWordOrThrow(arglist, "a workspaces directory"); break; - case NBCLIOptions.DOCKER_PROM_TAG: - arglist.removeFirst(); - this.docker_prom_tag = this.readWordOrThrow(arglist, "prometheus docker tag"); - break; - case NBCLIOptions.DOCKER_PROM_RETENTION_DAYS: - arglist.removeFirst(); - this.dockerPromRetentionDays = this.readWordOrThrow(arglist, "prometheus retention (3650d by default)"); - break; - case NBCLIOptions.DOCKER_GRAFANA_TAG: - arglist.removeFirst(); - this.docker_grafana_tag = this.readWordOrThrow(arglist, "grafana docker tag"); - break; case NBCLIOptions.VERSION: arglist.removeFirst(); this.wantsVersionShort = true; @@ -440,14 +413,6 @@ public class NBCLIOptions { arglist.removeFirst(); this.wantsVersionCoords = true; break; - case NBCLIOptions.DOCKER_METRICS_AT: - arglist.removeFirst(); - this.dockerMetricsHost = this.readWordOrThrow(arglist, "docker metrics host"); - break; - case NBCLIOptions.DOCKER_METRICS: - arglist.removeFirst(); - this.dockerMetrics = true; - break; case NBCLIOptions.SESSION_NAME: arglist.removeFirst(); this.sessionName = this.readWordOrThrow(arglist, "a session name"); @@ -794,14 +759,6 @@ public class NBCLIOptions { return this.enableChart; } - public boolean wantsDockerMetrics() { - return this.dockerMetrics; - } - - public String wantsDockerMetricsAt() { - return this.dockerMetricsHost; - } - public int getReportInterval() { return this.reportInterval; } @@ -947,14 +904,6 @@ public class NBCLIOptions { return this.wantsWorkloadsList; } - public String getDockerGrafanaTag() { - return this.docker_grafana_tag; - } - - public String getDockerPromTag() { - return this.docker_prom_tag; - } - public static class LoggerConfigData { public String file; public String pattern = ".*"; diff --git a/engine-cli/src/main/resources/argsfile.md b/engine-cli/src/main/resources/argsfile.md index 7803a29b6..d70774b19 100644 --- a/engine-cli/src/main/resources/argsfile.md +++ b/engine-cli/src/main/resources/argsfile.md @@ -28,13 +28,13 @@ variables. When the NBSTATEDIR location is first needed, the paths are checked in order, and the first one found is used. If one is not found on the filesystem, the first expanded value is used to create the state directory. - - + + If the default argsfile is is present, it is loaded by nosqlbench when it starts even if you don't ask it to. That is, nosqlbench behaves as if your first set of command line arguments is - - --argsfile-optional "$NBSTATEDIR/argsfile + + --argsfile-optional "$NBSTATEDIR/argsfile Just as with the NBSTATEDIR location, the argsfile can also be used like a search path. That is, if the value provided is a colon-delimited @@ -46,7 +46,6 @@ an argsfile when pinning options are used. An args file simply contains an argument on each line, like this: - --docker-metrics --annotate all --grafana-baseurl http://localhost:3000/ @@ -55,7 +54,7 @@ An args file simply contains an argument on each line, like this: It is possible to pin an option to the default args file by use of the `--pin` meta-option. This option will take the following command line argument and add it to the currently active args file. That means, if -you use `--pin --docker-metrics`, then `--docker-metrics` is added to +you use `--pin --annotate all`, then `--annotate all` is added to the args file. If there is an exact duplicate of the same option and value, then it is skipped, but if the option name is the same with a different value, then it is added at the end. This allows @@ -98,10 +97,10 @@ parameter values, then you must provide the value as well, as in To simply set global defaults, you can run nosqlbench with a command line like this: - ./nb --pin --docker-metrics-at metricsnode --pin --annotate all + ./nb --pin --annotate all # Compatibility You should use the --pin and --unpin options to make any changes to the argsfile when integrating or automating workflows. This ensures that - any changes to file format are handled for you by nb. \ No newline at end of file + any changes to file format are handled for you by nb. diff --git a/engine-cli/src/main/resources/basic.md b/engine-cli/src/main/resources/basic.md index 6c9465682..f7226c90e 100644 --- a/engine-cli/src/main/resources/basic.md +++ b/engine-cli/src/main/resources/basic.md @@ -19,12 +19,6 @@ # To provide your own contact points (comma separated), add the hosts= parameter ./nb cql-iot hosts=host1,host2 -# Additionally, if you have docker installed on your local system, -# and your user has permissions to use it, you can use -# --docker-metrics to stand up a live metrics dashboard at port 3000. - - ./nb cql-iot --docker-metrics - # Details on all command line options are available with ./nb help commandline diff --git a/engine-cli/src/main/resources/commandline.md b/engine-cli/src/main/resources/commandline.md index d200476b1..aceb161cb 100644 --- a/engine-cli/src/main/resources/commandline.md +++ b/engine-cli/src/main/resources/commandline.md @@ -198,10 +198,6 @@ a logfile will be created for this name. --session-name -Enlist nosqlbench to stand up your metrics infrastructure using a local docker runtime: - - --docker-metrics - When this option is set, nosqlbench will start graphite, prometheus, and grafana automatically on your local docker, configure them to work together, and point nosqlbench to send metrics the system automatically. It also imports a base dashboard for nosqlbench and configures grafana snapshot diff --git a/engine-cli/src/main/resources/docker-metrics.md b/engine-cli/src/main/resources/docker-metrics.md index 45392aa20..29b456bec 100644 --- a/engine-cli/src/main/resources/docker-metrics.md +++ b/engine-cli/src/main/resources/docker-metrics.md @@ -1,5 +1,11 @@ # docker-metrics +``` +NOTE: This feature has been removed and is no longer available. +``` + +--- + Enlist nosqlbench to stand up your metrics infrastructure using a local docker runtime: diff --git a/engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java b/engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java index 277e64fe0..698277425 100644 --- a/engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java +++ b/engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java @@ -76,14 +76,12 @@ public class RestHelper { if (resp.statusCode()==412) { logger.warn("Unable to configure dashboard, grafana precondition failed (status 412): " + resp.body()); String err = "When trying to configure grafana, any errors indicate that you may be trying to RE-configure an instance." + - " This may be a bug. If you already have a docker stack running, you can just use '--report-graphite-to localhost:9109'\n" + - " instead of --docker-metrics."; + " This may be a bug. If you already have a docker stack running, you can just use '--report-graphite-to localhost:9109'."; throw new BasicError(err); } else if (resp.statusCode()==401 && resp.body().contains("Invalid username")) { logger.warn("Unable to configure dashboard, grafana authentication failed (status " + resp.statusCode() + "): " + resp.body()); String err = "Grafana does not have the same password as expected for a new container. We shouldn't be trying to add dashboards on an" + - " existing container. This may be a bug. If you already have a docker stack running, you can just use '--report-graphite-to localhost:9109'" + - " instead of --docker-metrics."; + " existing container. This may be a bug. If you already have a docker stack running, you can just use '--report-graphite-to localhost:9109'."; throw new BasicError(err); } else if (resp.statusCode()<200 || resp.statusCode()>200) { logger.error("while trying to " + taskname +", received status code " + resp.statusCode() + " while trying to auto-configure grafana, with body:"); diff --git a/nbr/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java b/nbr/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java index 0254c2fcc..ca660833a 100644 --- a/nbr/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java +++ b/nbr/src/test/java/io/nosqlbench/engine/core/script/NBCliIntegrationTests.java @@ -46,31 +46,4 @@ public class NBCliIntegrationTests { System.out.println(result.getStderrData()); assertThat(result.exitStatus).isEqualTo(0); } - // disabled till after release - // This is not disabled on testbranch, only on main -// @Test -// public void dockerMetrics() { -// ProcessInvoker invoker = new ProcessInvoker(); -// invoker.setLogDir("logs/test"); -// -// // check for docker -// ProcessResult result = invoker.run("docker-detection-test", 15, -// "docker", "ps" -// ); -// -// System.out.println(result.getStdoutData()); -// System.out.println(result.getStderrData()); -// -// if(result.exitStatus ==0) { -// result = invoker.run("docker-metrics-test", 15, -// "java", "-jar", JARNAME, "--logs-dir", "logs/test", "--docker-metrics" -// ); -// System.out.println(result.getStdoutData()); -// System.out.println(result.getStderrData()); -// assertThat(result.exitStatus).isEqualTo(0); -// return; -// } -// -// logger.warn("skipped docker-metrics test because docker is not available"); -// } } diff --git a/pom.xml b/pom.xml index 81d72023d..f3bbafe88 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ engine-api engine-core engine-extensions - engine-docker + engine-clients engine-cli adapters-api