partial fixes for nosqlbench-797 Race condition between exceptional activity shutdown and normal scenario shutdown.

devdocs/concurrency/async_activity.puml

@@ -0,0 +1,105 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+title: Lifecycle of an activity
+
+control caller as caller
+control ActivityExecutor as ae
+control "Activity\nException\nHandler" as aeh
+control "Activity\nThread\nFactory" as atf
+control ExecutorService as aes
+control Annotator as ann
+control Activity as activity
+
+== startup sequence ==
+caller -> ae**: create
+  ae -> aeh**: create
+  ae -> atf**: create(\w Exception Handler)
+  aeh -> atf: <injected\nvia ctor>
+ ae -> aes**: create(\w Thread Factory)
+  atf -> aes: <injected\nvia ctor>
+
+caller -> ae: startActivity()
+activate ae
+ ae -> ann: Annotate Activity Start
+
+ ae -> activity: initActivity()
+ activate activity
+ ae <- activity
+ deactivate activity
+
+ note over ae,aes: align threadcount as explained below
+
+caller <- ae
+deactivate ae
+
+== dynamic threadcount update ==
+note over ae, aes: threads can be changed dynamically
+
+caller -> ae: apply params
+activate ae
+ ae->ae: align motor count
+ ae->aes: stop extra motors
+ ae->aes: <start missing motors>
+ group for each new thread/motor
+  ae -> aes: execute(<motor>)
+ activate aes
+  aes -> atf: get()
+  atf -> thread**: create
+  activate atf
+  aes <- atf: <thread>
+  deactivate atf
+  aes --> thread: run()
+  note over ann, thread: At this point, the\nmotor thread starts running\nthe defined activity's action\nover cycles
+  ae->ae: await thread state update
+
+ ae<-aes:
+ deactivate aes
+ end group
+caller <- ae
+deactivate ae
+
+== shutdown sequence [after startup] ==
+
+caller -> ae: stopActivity()
+activate ae
+
+ ae -> ae: request stop motors
+ ae -> ae: await all stop
+
+ ae -> activity: shutdownActivity()
+ activate activity
+ ae <- activity
+ deactivate activity
+
+ ae -> ann: Annotate Activity Finish
+
+caller <- ae
+deactivate ae
+
+== on exception in motor thread ==
+thread -> aeh: catch(<thrown exception>)
+aeh -> ae: notifyException\n(<thread>,<throwable>)
+activate ae
+ ae -> ae: save exception
+ ae -> ae: forceStopActivity()
+ ae -> aes: shutdown();
+ activate aes
+ ae <- aes:
+ deactivate aes
+
+ group if needed [after timeout]]
+  ae -> aes: shutdownNow();
+  activate aes
+  ae <- aes
+  deactivate aes
+ end group
+
+ ae -> activity: shutdownActivity();
+ ae -> activity: closeAutoCloseables();
+
+ note over thread: action\nthread\nterminates
+ destroy thread
+deactivate ae
+
+
+@enduml

devdocs/concurrency/async_scenario.puml

@@ -0,0 +1,67 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+
+title Lifecycle of a single scenario.call()
+control "caller" as c
+control "Scenario" as s
+control "Scenario\nController" as sc
+control "Scripting\nEngine" as engine
+control "Activity\nExecutor" as ae
+control "Java\nRuntime" as jrt
+control "Shutdown\nHook" as sh
+control "Annotations" as ann
+
+c -> s**: create
+
+c -> s: call()
+activate s
+
+ s -> sh**: create
+ s -> jrt: register(ShutdownHook)
+ s -> ann: Annotate Scenario Start
+
+ s -> sc**: create
+ s -> engine**: create
+
+ s -> engine: run(script)
+ activate engine
+ group async calls [javacript+Java]
+ engine <--> sc: scenario.(*)
+ engine <--> sc: activities.(*)
+ engine <--> sc: metrics.(*)
+ engine <--> sc: params.(*)
+ engine -> sc: start(<activity>)
+ activate sc
+ sc -> ae**: create
+ sc -> ae: startActivity()
+
+ deactivate sc
+ end group
+ s <- engine: result
+ deactivate engine
+
+ s -> sc: awaitCompletion()
+ activate sc
+  group for each activity
+   sc -> ae: awaitCompletion()
+   activate ae
+   sc <- ae
+   deactivate ae
+  end group
+
+ s <- sc
+ deactivate sc
+
+ s -> jrt: unregister(ShutdownHook)
+ s -> sh: run()
+ sh -> ann: Annotate Scenario Finish
+
+c <- s: Scenario\nResult
+deactivate s
+
+== on exception during call() ==
+ jrt -> sh: run()
+ sh -> ann: Annotate Scenario Finish
+
+
+@enduml

devdocs/concurrency/async_scenarios.puml

@@ -0,0 +1,62 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+
+title Lifecycle of Scenarios
+
+control "NBCLI" as nbcli
+control "Scenario" as s
+control "Scenario\nController" as sc
+control "Scenarios\nExecutor" as se
+control "Exception\nHandler" as seh
+control "Thread\nFactory" as stf
+control "Executor\nService" as ses
+
+nbcli -> se** : create
+ se -> seh** : create
+ se -> stf** : create ThreadFactory\n(w/ ExceptionHandler)
+ se -> ses** : create ExecutorService\n(w/ ThreadFactory)
+
+nbcli -> s** : create
+ s -> sc** : create
+nbcli --> se : execute(Scenario)
+se --> ses: submit(<Callable> Scenario)
+activate ses
+ ses -> future**: create
+se <-- ses: <Future<ScenarioResult>>
+deactivate ses
+
+== [async] on thread from thread factory ==
+ses -> stf: get()
+ stf -> thread**: create
+ses <- stf: <thread>
+ses -> thread: run task
+activate thread
+thread -> s: call()
+activate s
+thread <- s: ScenarioResult
+deactivate s
+thread -> future: result
+deactivate thread
+
+== [async] on NBCLI thread ==
+
+nbcli -> se: awaitAllResults();
+activate se
+ se -> ses: shutdown
+ loop timeout
+ se -> ses: awaitTermination(timeout)
+ activate ses
+ se <- ses
+ deactivate ses
+ end loop
+ loop each future
+ se -> future: get()
+ activate future
+ se <- future: ScenarioResult
+ deactivate future
+ end loop
+
+nbcli <- se: <ScenariosResults>
+deactivate se
+
+@enduml