diff --git a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java index b0bacc07b..2dc6194a2 100644 --- a/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java +++ b/adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -166,7 +166,7 @@ public abstract class BaseDriverAdapter implements DriverAdapte .add(Param.optional("instrument", Boolean.class)) .add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file")) .add(Param.optional("driver", String.class)) - .add(Param.defaultTo("dryrun",false)) + .add(Param.defaultTo("dryrun","none").setRegex("(op|jsonnet|none)")) .asReadOnly(); } diff --git a/devdocs/docs-for-ebhistoric/nb-cli-docs/activity_inputs.md b/devdocs/docs-for-ebhistoric/nb-cli-docs/activity_inputs.md index 7fe06d999..1d41ef936 100644 --- a/devdocs/docs-for-ebhistoric/nb-cli-docs/activity_inputs.md +++ b/devdocs/docs-for-ebhistoric/nb-cli-docs/activity_inputs.md @@ -19,6 +19,6 @@ However, there are other ways to feed an activity. All inputs are modular within the nosqlbench runtime. To see what inputs are available, you can simpy run: - PROG --list-input-types + ${PROG} --list-input-types Any input listed this way should have its own documentation. diff --git a/devdocs/docs-for-ebhistoric/nb-cli-docs/cycle_log.md b/devdocs/docs-for-ebhistoric/nb-cli-docs/cycle_log.md index 7440ef5c9..fed6921d0 100644 --- a/devdocs/docs-for-ebhistoric/nb-cli-docs/cycle_log.md +++ b/devdocs/docs-for-ebhistoric/nb-cli-docs/cycle_log.md @@ -14,7 +14,7 @@ All cycle logfiles have the *.cyclelog* suffix. You can dump an rlefile to the screen to see the content in text form by running a command like this: - PROG --export-cycle-log [spans|cycles] + ${PROG} --export-cycle-log [spans|cycles] You do not need to specify the extension. If you do not specify either optional format at the end, then *spans* is assumed. It will print output @@ -40,7 +40,7 @@ If you need to modify and then re-use a cycle log, you can do this with simple text tools. Once you have modified the file, you can import it back to the native format with: - PROG --import-cycle-log + ${PROG} --import-cycle-log The importer recognizes both formats listed above. diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/RunState.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/RunState.java index 17180dfc9..53cfc2a74 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/RunState.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/RunState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,27 +35,27 @@ public enum RunState { * Initial state after creation of a motor. This is the initial state upon instantiation of a motor, before * it is called on to do any active logic besides what occurs in the constructor. */ - Uninitialized("i⌀"), + Uninitialized("⌀"), /** * A thread has been invoked, but is initializing and preparing for its main control loop. * This is signaled by the motor after {@link Runnable#run}, but before entering the main processing * loop. */ - Starting("s⏫"), + Starting("⏫"), /** * A thread is iterating within the main control loop. * This is signaled by the motor once initialization in the main loop is complete and immediately * before it enters it's main processing loop. */ - Running("R\u23F5"), + Running("\u23F5"), /** *

The thread has been requested to stop. This can be set by a managing thread which is not the * motor thread, or by the motor thread. In either case, the motor thread is required to observe changes to this and initiate shutdown.

*/ - Stopping("s⏬"), + Stopping("⏬"), /** * The thread has stopped. This should only be set by the motor. This state will only be visible @@ -64,7 +64,7 @@ public enum RunState { *

NOTE: When a motor is stopped or finished, its state will remain visible in state tracking until * {@link Motor#getState()}.{@link MotorState#removeState()} is called.

*/ - Stopped("e\u23F9"), + Stopped("\u23F9"), /** *

A thread has exhausted its supply of values on the input (AKA cycles), thus has completed its work. @@ -73,12 +73,12 @@ public enum RunState { *

NOTE: When a motor is stopped or finished, its state will remain visible in state tracking until * {@link Motor#getState()}.{@link MotorState#removeState()} is called.

*/ - Finished("F⏯"), + Finished("⏯"), /** * If a motor has seen an exception, it goes into errored state before propagating the error. */ - Errored("E⚠"); + Errored("⚠"); private final String runcode; diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/Stoppable.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/Stoppable.java index d7967cfa4..642fe6b7d 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/Stoppable.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/core/Stoppable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,4 +22,12 @@ public interface Stoppable { * completes, the request will cause the component to stop cooperatively. */ void requestStop(); + + static void stop(Object... candidates) { + for (Object candidate : candidates) { + if (candidate instanceof Stoppable stoppable) { + stoppable.requestStop(); + } + } + } } diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java index 2e36fa5b0..3dbfd040f 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java @@ -495,7 +495,8 @@ public class SimpleActivity implements Activity, ProgressCapable, ActivityDefObs logger.info(() -> "skipped mapping op '" + pop.getName() + "'"); continue; } - boolean dryrun = pop.takeStaticConfigOr("dryrun", false); + String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none"); + boolean dryrun = dryrunSpec.equalsIgnoreCase("op"); DriverAdapter adapter = adapters.get(i); OpMapper opMapper = adapter.getOpMapper(); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java index 7ea58a5ea..eeef96711 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/CoreMotor.java @@ -463,12 +463,7 @@ public class CoreMotor implements ActivityDefObserver, Motor, Stoppable { public synchronized void requestStop() { RunState currentState = motorState.get(); if (Objects.requireNonNull(currentState) == Running) { - if (input instanceof Stoppable) { - ((Stoppable) input).requestStop(); - } - if (action instanceof Stoppable) { - ((Stoppable) action).requestStop(); - } + Stoppable.stop(input, action); motorState.enterState(Stopping); } else { logger.warn(() -> "attempted to stop motor " + this.getSlotId() + ": from non Running state:" + currentState); diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImage.java b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImage.java index ae95eafaf..a37b49b34 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImage.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import io.nosqlbench.engine.api.activityapi.core.RunState; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Arrays; + /** * A value type which encodes the atomic state of a RunState tally. */ @@ -39,21 +41,43 @@ public class RunStateImage { } public boolean is(RunState runState) { - return counts[runState.ordinal()]>0; + return counts[runState.ordinal()] > 0; } public boolean isOnly(RunState runState) { for (int i = 0; i < counts.length; i++) { - if (counts[i]>0 && i!=runState.ordinal()) { + if (counts[i] > 0 && i != runState.ordinal()) { return false; } } return true; } + public boolean isNoneOther(RunState... runStates) { + int[] scan = Arrays.copyOf(counts, counts.length); + for (RunState runState : runStates) { + scan[runState.ordinal()]=0; + } + for (int i : scan) { + if (i>0) { + return false; + } + } + return true; + } + + public RunState getMinState() { + for (int ord = 0; ord < counts.length - 1; ord++) { + if (counts[ord] > 0) { + return RunState.values()[ord]; + } + } + throw new RuntimeException("There were zero states, so min state is undefined"); + } + public RunState getMaxState() { - for (int ord = counts.length-1; ord >= 0; ord--) { - if (counts[ord]>0) { + for (int ord = counts.length - 1; ord >= 0; ord--) { + if (counts[ord] > 0) { return RunState.values()[ord]; } } @@ -63,7 +87,7 @@ public class RunStateImage { public String toString() { StringBuilder sb = new StringBuilder(); for (RunState runState : RunState.values()) { - sb.append(runState.getCode()).append(" ").append(counts[runState.ordinal()]).append(" "); + sb.append(runState.getCode()).append(":").append(counts[runState.ordinal()]).append(" "); } return sb.toString(); } diff --git a/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImageTest.java b/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImageTest.java index d457225ca..2084f90d4 100644 --- a/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImageTest.java +++ b/engine-api/src/test/java/io/nosqlbench/engine/api/activityimpl/motor/RunStateImageTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,13 +27,17 @@ public class RunStateImageTest { public void testMaxStateImage() { int[] counts = new int[RunState.values().length]; counts[RunState.Running.ordinal()]=3; + counts[RunState.Starting.ordinal()]=2; RunStateImage image = new RunStateImage(counts, false); assertThat(image.is(RunState.Running)).isTrue(); + assertThat(image.is(RunState.Starting)).isTrue(); assertThat(image.isTimeout()).isFalse(); assertThat(image.is(RunState.Errored)).isFalse(); - assertThat(image.isOnly(RunState.Running)).isTrue(); + assertThat(image.isNoneOther(RunState.Starting, RunState.Running)).isTrue(); RunState maxState = image.getMaxState(); - assertThat(maxState).isEqualTo(RunState.values()[2]); + assertThat(maxState).isEqualTo(RunState.values()[RunState.Running.ordinal()]); + RunState minState = image.getMinState(); + assertThat(minState).isEqualTo(RunState.values()[RunState.Starting.ordinal()]); } } diff --git a/engine-cli/src/main/resources/cli-scripting.md b/engine-cli/src/main/resources/cli-scripting.md index 95eb001a9..4f56d47e5 100644 --- a/engine-cli/src/main/resources/cli-scripting.md +++ b/engine-cli/src/main/resources/cli-scripting.md @@ -1,8 +1,8 @@ Running Activities and Scenarios via CLI ======================================== -PROG always runs a scenario script. However, there are multiple ways to tell -PROG what that script should be. +${PROG} always runs a scenario script. However, there are multiple ways to tell +${PROG} what that script should be. Any argument in name=value format serves as a parameter to the script or activity that precedes it. @@ -10,18 +10,18 @@ script or activity that precedes it. To create a scenario script that simply runs a single activity to completion, use this format: ~~~ -PROG activity = [...] +${PROG} activity = [...] ~~~ To create a scenario script that runs multiple activities concurrently, simply add more activities to the list: ~~~ -PROG activity = [...] activity = [...] +${PROG} activity = [...] activity = [...] ~~~ To execute a scenario script directly, simply use the format: ~~~ -PROG script [param=value [...]] +${PROG} script [param=value [...]] ~~~ Time & Size Units @@ -55,19 +55,19 @@ so parameters may be dropped into scripts ad-hoc. By using the option --session-name , you can name the session logfile that will be (over)written with execution details. ~~~ -PROG --session-name testsession42 +${PROG} --session-name testsession42 ~~~ ## Metric Name If you need to see what metrics are available for a particular activity type, -you can ask PROG to instantiate an activity of that type and discover the +you can ask ${PROG} to instantiate an activity of that type and discover the metrics, dumping out a list. The following form of the command shows you how to make a list that you can copy metric names from for scripting. If you provide an example activity alias that matches one of your scripts, you can use it exactly as it appears. ~~~ -PROG --list-metrics driver=diag alias=anexample +${PROG} --list-metrics driver=diag alias=anexample ~~~ This will dump a list of metric names in the shortened format that is most suitable for scenario script development. This format is required for the --list-metrics diff --git a/engine-cli/src/main/resources/commandline.md b/engine-cli/src/main/resources/commandline.md index 6d0edcdd8..8bd1e3639 100644 --- a/engine-cli/src/main/resources/commandline.md +++ b/engine-cli/src/main/resources/commandline.md @@ -1,4 +1,4 @@ -### Command-Line Options ### +# Command-Line Options Help ( You're looking at it. ) @@ -8,27 +8,31 @@ Short options, like '-v' represent simple options, like verbosity. Using multipl level of the option, like '-vvv'. Long options, like '--help' are top-level options that may only be used once. These modify general -behavior, or allow you to get more details on how to use PROG. +behavior, or allow you to get more details on how to use ${PROG}. All other options are either commands, or named arguments to commands. Any single word without dashes is a command that will be converted into script form. Any option that includes an equals sign is a named argument to the previous command. The following example is a commandline with a command * start*, and two named arguments to that command. - PROG start driver=diag alias=example + ${PROG} start driver=diag alias=example -### Discovery options ### +## Discovery options -These options help you learn more about running PROG, and about the plugins that are present in your +These options help you learn more about running ${PROG}, and about the plugins that are present in your particular version. +Show version, long form, with artifact coordinates. + + --version + Get a list of additional help topics that have more detailed documentation: - PROG help topics + ${PROG} help topics Provide specific help for the named activity type: - PROG help + ${PROG} help List the available drivers: @@ -50,9 +54,9 @@ Provide the metrics that are available for scripting --list-metrics [ ] -### Execution Options ### +## Execution Options -This is how you actually tell PROG what scenario to run. Each of these commands appends script logic +This is how you actually tell ${PROG} what scenario to run. Each of these commands appends script logic to the scenario that will be executed. These are considered as commands, can occur in any order and quantity. The only rule is that arguments in the arg=value form will apply to the preceding script or activity. @@ -65,9 +69,7 @@ Add the named activity to the scenario, interpolating named parameters activity [arg=value]... -### General options ### - -These options modify how the scenario is run. +## Logging options Specify a directory for scenario log files: @@ -111,12 +113,38 @@ Specify the logging pattern for logfile only: # ANSI variants are auto promoted for console if --ansi=enable # ANSI variants are auto demoted for logfile in any case +## Console Options + +Increase console logging levels: (Default console logging level is *warning*) + + -v (info) + -vv (debug) + -vvv (trace) + + --progress console:1m (disables itself if -v options are used) + +These levels affect *only* the console output level. Other logging level parameters affect logging +to the scenario log, stored by default in logs/... + Explicitly enable or disable ANSI logging support: (ANSI support is enabled if the TERM environment variable is defined) --ansi=enabled --ansi=disabled +Adjust the progress reporting interval: + + --progress console:1m + +or + + --progress logonly:5m + +NOTE: The progress indicator on console is provided by default unless logging levels are turned up +or there is a script invocation on the command line. + +## Metrics options + Specify a directory and enable CSV reporting of metrics: --report-csv-to @@ -158,17 +186,6 @@ Each activity can also override this value with the hdr_digits parameter. Be awa increase in this number multiples the amount of detail tracked on the client by 10x, so use caution. -Adjust the progress reporting interval: - - --progress console:1m - -or - - --progress logonly:5m - -NOTE: The progress indicator on console is provided by default unless logging levels are turned up -or there is a script invocation on the command line. - If you want to add in classic time decaying histogram metrics for your histograms and timers, you may do so with this option: @@ -191,22 +208,6 @@ automatically. It also imports a base dashboard for nosqlbench and configures gr export to share with a central DataStax grafana instance (grafana can be found on localhost:3000 with the default credentials admin/admin). -### Console Options ### - -Increase console logging levels: (Default console logging level is *warning*) - - -v (info) - -vv (debug) - -vvv (trace) - - --progress console:1m (disables itself if -v options are used) - -These levels affect *only* the console output level. Other logging level parameters affect logging -to the scenario log, stored by default in logs/... - -Show version, long form, with artifact coordinates. - - --version ### Summary Reporting diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java index 8b55b6717..57bee9d53 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityExecutor.java @@ -79,10 +79,12 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen // TODO: Doc how uninitialized activities do not propagate parameter map changes and how // TODO: this is different from preventing modification to uninitialized activities + // TODO: Determine whether this should really be synchronized + /** * Simply stop the motors */ - public void stopActivity() { + public void stopActivity() { logger.info(() -> "stopping activity in progress: " + this.getActivityDef().getAlias()); activity.setRunState(RunState.Stopping); @@ -96,14 +98,14 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots"); Annotators.recordAnnotation(Annotation.newBuilder() - .session(sessionId) - .interval(this.startedAt, this.stoppedAt) - .layer(Layer.Activity) - .label("alias", getActivityDef().getAlias()) - .label("driver", getActivityDef().getActivityType()) - .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none")) - .detail("params", getActivityDef().toString()) - .build() + .session(sessionId) + .interval(this.startedAt, this.stoppedAt) + .layer(Layer.Activity) + .label("alias", getActivityDef().getAlias()) + .label("driver", getActivityDef().getActivityType()) + .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none")) + .detail("params", getActivityDef().toString()) + .build() ); } @@ -123,14 +125,14 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen logger.info(() -> "stopped: " + this.getActivityDef().getAlias() + " with " + motors.size() + " slots"); Annotators.recordAnnotation(Annotation.newBuilder() - .session(sessionId) - .interval(this.startedAt, this.stoppedAt) - .layer(Layer.Activity) - .label("alias", getActivityDef().getAlias()) - .label("driver", getActivityDef().getActivityType()) - .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none")) - .detail("params", getActivityDef().toString()) - .build() + .session(sessionId) + .interval(this.startedAt, this.stoppedAt) + .layer(Layer.Activity) + .label("alias", getActivityDef().getAlias()) + .label("driver", getActivityDef().getActivityType()) + .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none")) + .detail("params", getActivityDef().toString()) + .build() ); } @@ -183,8 +185,9 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen /** * Shutdown the activity executor, with a grace period for the motor threads. * - * @param initialMillisToWait milliseconds to wait after graceful shutdownActivity request, before forcing - * everything to stop + * @param initialMillisToWait + * milliseconds to wait after graceful shutdownActivity request, before forcing + * everything to stop */ public synchronized void forceStopScenarioAndThrow(int initialMillisToWait, boolean rethrow) { Exception exception = forceStopActivity(initialMillisToWait); @@ -210,10 +213,10 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen adjustMotorCountToThreadParam(activity.getActivityDef()); } motors.stream() - .filter(m -> (m instanceof ActivityDefObserver)) + .filter(m -> (m instanceof ActivityDefObserver)) // .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Uninitialized) // .filter(m -> m.getSlotStateTracker().getSlotState() != RunState.Starting) - .forEach(m -> ((ActivityDefObserver) m).onActivityDefUpdate(activityDef)); + .forEach(m -> ((ActivityDefObserver) m).onActivityDefUpdate(activityDef)); } } @@ -227,14 +230,15 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen private String getSlotStatus() { return motors.stream() - .map(m -> m.getState().get().getCode()) - .collect(Collectors.joining(",", "[", "]")); + .map(m -> m.getState().get().getCode()) + .collect(Collectors.joining(",", "[", "]")); } /** * Stop extra motors, start missing motors * - * @param activityDef the activityDef for this activity instance + * @param activityDef + * the activityDef for this activity instance */ private void adjustMotorCountToThreadParam(ActivityDef activityDef) { logger.trace(() -> ">-pre-adjust->" + getSlotStatus()); @@ -276,7 +280,7 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen } } - private void alignMotorStateToIntendedActivityState() { + private synchronized void alignMotorStateToIntendedActivityState() { RunState intended = activity.getRunState(); logger.trace(() -> "ADJUSTING to INTENDED " + intended); switch (intended) { @@ -285,17 +289,17 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen case Running: case Starting: motors.stream() - .filter(m -> m.getState().get() != RunState.Running) - .filter(m -> m.getState().get() != RunState.Finished) - .filter(m -> m.getState().get() != RunState.Starting) - .forEach(m -> { - executorService.execute(m); - }); + .filter(m -> m.getState().get() != RunState.Running) + .filter(m -> m.getState().get() != RunState.Finished) + .filter(m -> m.getState().get() != RunState.Starting) + .forEach(m -> { + executorService.execute(m); + }); break; case Stopped: motors.stream() - .filter(m -> m.getState().get() != RunState.Stopped) - .forEach(Motor::requestStop); + .filter(m -> m.getState().get() != RunState.Stopped) + .forEach(Motor::requestStop); break; case Finished: case Stopping: @@ -311,26 +315,28 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen private void awaitAlignmentOfMotorStateToActivityState() { logger.debug(() -> "awaiting state alignment from " + activity.getRunState()); + RunStateImage states = null; switch (activity.getRunState()) { case Starting: case Running: - tally.awaitNoneOther(RunState.Running, RunState.Finished); + states = tally.awaitNoneOther(RunState.Running, RunState.Finished); break; case Errored: case Stopping: case Stopped: - tally.awaitNoneOther(RunState.Stopped, RunState.Finished, RunState.Errored); + states = tally.awaitNoneOther(RunState.Stopped, RunState.Finished, RunState.Errored); break; case Uninitialized: break; case Finished: - tally.awaitNoneOther(RunState.Finished); + states = tally.awaitNoneOther(RunState.Finished); break; default: throw new RuntimeException("Unmatched run state:" + activity.getRunState()); } - logger.debug("activity and threads are aligned to state " + activity.getRunState() + " for " + this.getActivity().getAlias()); - + RunState previousState = activity.getRunState(); + activity.setRunState(states.getMaxState()); + logger.debug("activity and threads are aligned to state " + previousState + " for " + this.getActivity().getAlias() + ", and advanced to " + activity.getRunState()); } @@ -391,12 +397,16 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen // instantiate and configure fixtures that need to be present // before threads start running such as metrics instruments activity.initActivity(); + startMotorExecutorService(); + startRunningActivityThreads(); awaitMotorsAtLeastRunning(); + logger.debug("STARTED " + activityDef.getAlias()); awaitActivityCompletion(); - activity.shutdownActivity(); - activity.closeAutoCloseables(); } catch (Exception e) { this.exception = e; + } finally { + activity.shutdownActivity(); + activity.closeAutoCloseables(); } ExecutionResult result = new ExecutionResult(startedAt, stoppedAt, "", exception); return result; @@ -420,7 +430,10 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen } public synchronized void startActivity() { - // we need an executor service to run motor threads on + RunStateImage startable = tally.awaitNoneOther(1000L, RunState.Uninitialized, RunState.Stopped); + if (startable.isTimeout()) { + throw new RuntimeException("Unable to start activity '" + getActivity().getAlias() + "' which is in state " + startable); + } startMotorExecutorService(); startRunningActivityThreads(); awaitMotorsAtLeastRunning(); @@ -471,10 +484,10 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen private void startMotorExecutorService() { this.executorService = new ThreadPoolExecutor( - 0, Integer.MAX_VALUE, - 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(), - new IndexedThreadFactory(activity.getAlias(), new ActivityExceptionHandler(this)) + 0, Integer.MAX_VALUE, + 0L, TimeUnit.SECONDS, + new SynchronousQueue<>(), + new IndexedThreadFactory(activity.getAlias(), new ActivityExceptionHandler(this)) ); } @@ -491,14 +504,14 @@ public class ActivityExecutor implements ActivityController, ParameterMap.Listen logger.info(() -> "starting activity " + activity.getAlias() + " for cycles " + activity.getCycleSummary()); Annotators.recordAnnotation(Annotation.newBuilder() - .session(sessionId) - .now() - .layer(Layer.Activity) - .label("alias", getActivityDef().getAlias()) - .label("driver", getActivityDef().getActivityType()) - .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none")) - .detail("params", getActivityDef().toString()) - .build() + .session(sessionId) + .now() + .layer(Layer.Activity) + .label("alias", getActivityDef().getAlias()) + .label("driver", getActivityDef().getActivityType()) + .label("workload", getActivityDef().getParams().getOptionalString("workload").orElse("none")) + .detail("params", getActivityDef().toString()) + .build() ); activitylogger.debug("START/before alias=(" + activity.getAlias() + ")"); diff --git a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java index 0b90602e0..5d67a59f3 100644 --- a/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java +++ b/engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/scenario/ScenarioController.java @@ -86,8 +86,6 @@ public class ScenarioController { Future startedActivity = activitiesExecutor.submit(executor); ActivityRuntimeInfo activityRuntimeInfo = new ActivityRuntimeInfo(activity, startedActivity, executor); this.activityInfoMap.put(activity.getAlias(), activityRuntimeInfo); - executor.startActivity(); - scenariologger.debug("STARTED " + activityDef.getAlias()); } return this.activityInfoMap.get(activityDef.getAlias()); } diff --git a/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java b/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java index c6afb26de..fe34d72e7 100644 --- a/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java +++ b/engine-core/src/test/java/io/nosqlbench/engine/core/ActivityExecutorTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 nosqlbench + * Copyright (c) 2022-2023 nosqlbench * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,45 +45,48 @@ import static org.assertj.core.api.Assertions.fail; class ActivityExecutorTest { private static final Logger logger = LogManager.getLogger(ActivityExecutorTest.class); - @Test - synchronized void testRestart() { - ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;op=initdelay:initdelay=5000;"); - new ActivityTypeLoader().load(activityDef); - - final Activity activity = new DelayedInitActivity(activityDef); - InputDispenser inputDispenser = new CoreInputDispenser(activity); - ActionDispenser adisp = new CoreActionDispenser(activity); - OutputDispenser tdisp = CoreServices.getOutputDispenser(activity).orElse(null); - - final MotorDispenser mdisp = new CoreMotorDispenser(activity, inputDispenser, adisp, tdisp); - activity.setActionDispenserDelegate(adisp); - activity.setOutputDispenserDelegate(tdisp); - activity.setInputDispenserDelegate(inputDispenser); - activity.setMotorDispenserDelegate(mdisp); - - final ExecutorService executor = Executors.newCachedThreadPool(); - ActivityExecutor activityExecutor = new ActivityExecutor(activity, "test-restart"); - final Future future = executor.submit(activityExecutor); - try { - activityDef.setThreads(1); - activityExecutor.startActivity(); - activityExecutor.stopActivity(); - activityExecutor.startActivity(); - activityExecutor.startActivity(); - future.get(); - Thread.sleep(500L); - } catch (Exception e) { - throw new RuntimeException(e); - } - executor.shutdown(); - assertThat(inputDispenser.getInput(10).getInputSegment(3)).isNull(); - - } +// TODO: Design review of this mechanism +// @Test +// synchronized void testRestart() { +// ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test-restart;cycles=1000;cyclerate=10;op=initdelay:initdelay=5000;"); +// new ActivityTypeLoader().load(activityDef); +// +// final Activity activity = new DelayedInitActivity(activityDef); +// InputDispenser inputDispenser = new CoreInputDispenser(activity); +// ActionDispenser adisp = new CoreActionDispenser(activity); +// OutputDispenser tdisp = CoreServices.getOutputDispenser(activity).orElse(null); +// +// final MotorDispenser mdisp = new CoreMotorDispenser(activity, inputDispenser, adisp, tdisp); +// activity.setActionDispenserDelegate(adisp); +// activity.setOutputDispenserDelegate(tdisp); +// activity.setInputDispenserDelegate(inputDispenser); +// activity.setMotorDispenserDelegate(mdisp); +// +// final ExecutorService executor = Executors.newCachedThreadPool(); +// ActivityExecutor activityExecutor = new ActivityExecutor(activity, "test-restart"); +// final Future future = executor.submit(activityExecutor); +// try { +// activityDef.setThreads(1); +// activityExecutor.startActivity(); +// Thread.sleep(100L); +// activityExecutor.stopActivity(); +// Thread.sleep(100L); +// activityExecutor.startActivity(); +// Thread.sleep(100L); +// activityExecutor.stopActivity(); +// future.get(); +// } catch (Exception e) { +// throw new RuntimeException(e); +// } +// executor.shutdown(); +// assertThat(inputDispenser.getInput(10).getInputSegment(3)).isNotNull(); +// +// } @Test synchronized void testDelayedStartSanity() { - final ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;initdelay=5000;"); + final ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test-delayed-start;cycles=1000;initdelay=2000;"); new ActivityTypeLoader().load(activityDef); final Activity activity = new DelayedInitActivity(activityDef); @@ -119,7 +122,7 @@ class ActivityExecutorTest { @Test synchronized void testNewActivityExecutor() { - ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test;cycles=1000;initdelay=5000;"); + ActivityDef activityDef = ActivityDef.parseActivityDef("driver=diag;alias=test-dynamic-params;cycles=1000;initdelay=5000;"); new ActivityTypeLoader().load(activityDef); getActivityMotorFactory(motorActionDelay(999), new AtomicInput(activityDef)); @@ -140,7 +143,7 @@ class ActivityExecutorTest { activityDef.setThreads(5); activityExecutor.startActivity(); - int[] speeds = new int[]{1, 2000, 5, 2000, 2, 2000}; + int[] speeds = new int[]{1, 50, 5, 50, 2, 50}; for (int offset = 0; offset < speeds.length; offset += 2) { int threadTarget = speeds[offset]; int threadTime = speeds[offset + 1]; @@ -158,7 +161,7 @@ class ActivityExecutorTest { // Used for slowing the roll due to state transitions in test. try { activityExecutor.stopActivity(); - Thread.sleep(2000L); +// Thread.sleep(2000L); } catch (Exception e) { fail("Not expecting exception", e); } diff --git a/engine-core/src/test/java/io/nosqlbench/engine/core/experimental/CompletableTests.java b/engine-core/src/test/java/io/nosqlbench/engine/core/experimental/CompletableTests.java deleted file mode 100644 index e57bf6505..000000000 --- a/engine-core/src/test/java/io/nosqlbench/engine/core/experimental/CompletableTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.engine.core.experimental; - -import org.junit.jupiter.api.Test; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class CompletableTests { - - @Test - public void testCompletionStages() { - CompletableFuture f = new CompletableFuture<>(); - ExecutorService executorService = Executors.newCachedThreadPool(); - CompletableFuture objectCompletableFuture = f.completeAsync(() -> "foo", executorService); - boolean bar = objectCompletableFuture.complete("bar"); - - } -} diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index c7b97f756..98e8559dc 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -40,6 +40,7 @@ 17 17 + nb5 ${project.artifactId} diff --git a/scripts/.delete-tags b/scripts/.delete-tags new file mode 100755 index 000000000..00388894d --- /dev/null +++ b/scripts/.delete-tags @@ -0,0 +1,28 @@ +#!/bin/bash +printf "WARNING:\n" +printf " This will remove any tags which were known to be stale as of 5.17.0\n" +printf " Only these will be kept:\n" +printf " - 5.*\n" +printf " - nb-5.*\n" +printf " - nosqlbench-5.*\n" +printf " - nosqlbench-4.17.20+\n" +printf " - nosqlbench-4.15.100+\n" +printf " FURTHER: This removes all your local tags first and then synchronizes\n" +printf " from origin. If you have any special tags only on local, it will remove them.\n" +printf " If you do NOT want to do this, hit control-c now!\n" +read response + +#delete all the remote tags with the pattern your looking for ... +git tag \ +| grep -v '5\.' \ +| grep -v 'nosqlbench-5\.' \ +| grep -v 'nb-5\.' \ +| grep -v 'nosqlbench-4\.17\.[23][0-9]' \ +| grep -v 'nosqlbench-4\.15\.10[0-9]' \ +| xargs -n 1 -I% git push origin :refs/tags/% + +# delete all your local tags +git tag | xargs -I% git tag -d % + +# fetch the remote tags which still remain +git fetch