add dryrun=emit support

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java

@@ -99,7 +99,7 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
         this.metrics = metrics;
     }
 
-    public final List<Row> apply(long cycle) {
+    public final ArrayList<Row> apply(long cycle) {
 
         Statement<?> statement = getStmt();
         logger.trace(() -> "apply() invoked, statement obtained, executing async with page size: " + statement.getPageSize() + " thread local rows: ");
@@ -118,7 +118,7 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
         });
 
         try {
-            return rowsStage.toCompletableFuture().get(300, TimeUnit.SECONDS);
+            return new PrintableRowList(rowsStage.toCompletableFuture().get(300, TimeUnit.SECONDS));
         } catch (ExecutionException exe) {
             Throwable ee = exe.getCause();
             if (ee instanceof RuntimeException re) {
@@ -142,6 +142,21 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
 //            processors.flush();
     }
 
+    private static class PrintableRowList extends ArrayList<Row> {
+        public PrintableRowList(List<Row> values) {
+            super(values);
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            for (Row row : this) {
+                sb.append(row.getFormattedContents());
+            }
+            return sb.toString();
+        }
+    }
+
     //    private BiFunction<AsyncResultSet,Throwable> handler
     @Override
     public Op getNextOp() {

adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/BaseDriverAdapter.java

@@ -176,7 +176,7 @@ public abstract class BaseDriverAdapter<R extends Op, S> extends NBBaseComponent
                 .add(Param.optional("instrument", Boolean.class))
                 .add(Param.optional(List.of("workload", "yaml"), String.class, "location of workload yaml file"))
                 .add(Param.optional("driver", String.class))
-                .add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|none)"))
+                .add(Param.defaultTo("dryrun", "none").setRegex("(op|jsonnet|emit|none)"))
                 .add(Param.optional("maxtries", Integer.class))
                 .asReadOnly();
     }

adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/EmitterOp.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapters.api.activityimpl.uniform;
+
+import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
+
+public class EmitterOp implements CycleOp<Object> {
+
+    private final CycleOp<?> cycleOp;
+    public EmitterOp(CycleOp<?> cycleOp) {
+        this.cycleOp = cycleOp;
+    }
+
+    @Override
+    public Object apply(long value) {
+            Object result = cycleOp.apply(value);
+            System.out.println("result from cycle " + value + ":\n"+result);
+            return result;
+    }
+}

adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/EmitterOpDispenserWrapper.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022-2023 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapters.api.activityimpl.uniform;
+
+import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
+import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
+import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
+import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
+import io.nosqlbench.adapters.api.templating.ParsedOp;
+
+public class EmitterOpDispenserWrapper extends BaseOpDispenser<Op, Object> {
+
+    private final OpDispenser<? extends CycleOp<?>> realDispenser;
+
+    public EmitterOpDispenserWrapper(DriverAdapter<Op,Object> adapter, ParsedOp pop, OpDispenser<? extends CycleOp<?>> realDispenser) {
+        super(adapter, pop);
+        this.realDispenser = realDispenser;
+    }
+    @Override
+    public EmitterOp apply(long cycle) {
+        CycleOp<?> cycleOp = realDispenser.apply(cycle);
+        return new EmitterOp(cycleOp);
+    }
+}

engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java

@@ -16,6 +16,8 @@
 
 package io.nosqlbench.engine.api.activityimpl;
 
+import io.nosqlbench.adapters.api.activityimpl.uniform.EmitterOpDispenserWrapper;
+import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
 import io.nosqlbench.nb.api.components.core.NBComponent;
 import io.nosqlbench.nb.api.components.core.NBBaseComponent;
 import io.nosqlbench.nb.api.components.events.ParamChange;
@@ -56,12 +58,10 @@ import org.apache.logging.log4j.Logger;
 
 import java.io.InputStream;
 import java.io.PrintWriter;
-import java.lang.reflect.AnnotatedType;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.function.Function;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
 
 /**
  * A default implementation of an Activity, suitable for building upon.
@@ -71,7 +71,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
 
     protected ActivityDef activityDef;
     private final List<AutoCloseable> closeables = new ArrayList<>();
-    private MotorDispenser motorDispenser;
+    private MotorDispenser<?> motorDispenser;
     private InputDispenser inputDispenser;
     private ActionDispenser actionDispenser;
     private OutputDispenser markerDispenser;
@@ -90,7 +90,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
     private final RunStateTally tally = new RunStateTally();
 
     public SimpleActivity(NBComponent parent, ActivityDef activityDef) {
-        super(parent,NBLabels.forKV("activity",activityDef.getAlias()).and(activityDef.auxLabels()));
+        super(parent, NBLabels.forKV("activity", activityDef.getAlias()).and(activityDef.auxLabels()));
         this.activityDef = activityDef;
         if (activityDef.getAlias().equals(ActivityDef.DEFAULT_ALIAS)) {
             Optional<String> workloadOpt = activityDef.getParams().getOptionalString(
@@ -109,7 +109,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
     }
 
     public SimpleActivity(NBComponent parent, String activityDefString) {
-        this(parent,ActivityDef.parseActivityDef(activityDefString));
+        this(parent, ActivityDef.parseActivityDef(activityDefString));
     }
 
     @Override
@@ -121,7 +121,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
         if (null == this.errorHandler) {
             errorHandler = new NBErrorHandler(
                 () -> activityDef.getParams().getOptionalString("errors").orElse("stop"),
-                    () -> getExceptionMetrics());
+                this::getExceptionMetrics);
         }
         return errorHandler;
     }
@@ -200,7 +200,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
     }
 
     public String toString() {
-        return (activityDef!=null ? activityDef.getAlias() : "unset_alias" )  + ':' + this.runState + ':' + this.tally ;
+        return (activityDef != null ? activityDef.getAlias() : "unset_alias") + ':' + this.runState + ':' + this.tally;
     }
 
     @Override
@@ -316,6 +316,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
     public void createOrUpdateStrideLimiter(SimRateSpec spec) {
         strideLimiter = RateLimiters.createOrUpdate(this, strideLimiter, spec);
     }
+
     public void createOrUpdateCycleLimiter(SimRateSpec spec) {
         cycleLimiter = RateLimiters.createOrUpdate(this, cycleLimiter, spec);
     }
@@ -409,24 +410,23 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
                 " more information.");
         }
 
-        if (0 < this.activityDef.getCycleCount() && 0 == seq.getOps().size()) {
+        if (0 < this.activityDef.getCycleCount() && seq.getOps().isEmpty()) {
             throw new BasicError("You have configured a zero-length sequence and non-zero cycles. Tt is not possible to continue with this activity.");
         }
     }
 
 
     protected <O extends Op> OpSequence<OpDispenser<? extends O>> createOpSourceFromParsedOps(
-            Map<String, DriverAdapter> adapterCache,
+//        Map<String, DriverAdapter<?,?>> adapterCache,
-            Map<String, OpMapper<Op>> mapperCache,
+//        Map<String, OpMapper<? extends Op>> mapperCache,
-            List<DriverAdapter> adapters,
+        List<DriverAdapter<?,?>> adapters,
         List<ParsedOp> pops
     ) {
         try {
 
             List<Long> ratios = new ArrayList<>(pops.size());
 
-            for (int i = 0; i < pops.size(); i++) {
+            for (ParsedOp pop : pops) {
-                ParsedOp pop = pops.get(i);
                 long ratio = pop.takeStaticConfigOr("ratio", 1);
                 ratios.add(ratio);
             }
@@ -445,16 +445,21 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
                     logger.info(() -> "skipped mapping op '" + pop.getName() + '\'');
                     continue;
                 }
-                String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
-                boolean dryrun = "op".equalsIgnoreCase(dryrunSpec);
 
-                DriverAdapter adapter = adapters.get(i);
+                DriverAdapter<?,?> adapter = adapters.get(i);
-                OpMapper opMapper = adapter.getOpMapper();
+                OpMapper<? extends Op> opMapper = adapter.getOpMapper();
                 OpDispenser<? extends Op> dispenser = opMapper.apply(pop);
 
-                if (dryrun) {
+                String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
-                    dispenser = new DryRunOpDispenserWrapper(adapter, pop, dispenser);
+                if ("op".equalsIgnoreCase(dryrunSpec)) {
+                    dispenser = new DryRunOpDispenserWrapper((DriverAdapter<Op,Object>)adapter, pop, dispenser);
                     dryrunCount++;
+                } else if ("emit".equalsIgnoreCase(dryrunSpec)) {
+                    dispenser = new EmitterOpDispenserWrapper(
+                        (DriverAdapter<Op,Object>)adapter,
+                        pop,
+                        (OpDispenser<? extends CycleOp<?>>) dispenser
+                    );
                 }
 
 //                if (strict) {
@@ -475,7 +480,7 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
 
     }
 
-    protected List<OpTemplate> loadOpTemplates(Optional<DriverAdapter<?,?>> defaultDriverAdapter) {
+    protected List<OpTemplate> loadOpTemplates(DriverAdapter<?, ?> defaultDriverAdapter) {
 
         String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
 
@@ -493,11 +498,11 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
                 throw new BasicError("There were no active op templates with tag filter '"
                     + tagfilter + "', since all " + unfilteredOps.size() + " were filtered out.");
             }
-            if (defaultDriverAdapter.isPresent() && defaultDriverAdapter.get() instanceof SyntheticOpTemplateProvider sotp) {
+            if (defaultDriverAdapter instanceof SyntheticOpTemplateProvider sotp) {
                 filteredOps = sotp.getSyntheticOpTemplates(opsDocList, this.activityDef.getParams());
                 Objects.requireNonNull(filteredOps);
                 if (filteredOps.isEmpty()) {
-                    throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.get().getAdapterName() + '\'' +
+                    throw new BasicError("Attempted to create synthetic ops from driver '" + defaultDriverAdapter.getAdapterName() + '\'' +
                         " but no ops were created. You must provide either a workload or an op parameter. Activities require op templates.");
                 }
             } else {
@@ -507,28 +512,28 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
                     2) op='inline template'
                     3) driver=stdout (or any other drive that can synthesize ops)""");
             }
-            if (filteredOps.isEmpty()) {
-                throw new BasicError("There were no active op templates with tag filter '" + tagfilter + '\'');
-            }
-        }
-
-        if (filteredOps.isEmpty()) {
-            throw new OpConfigError("No op templates found. You must provide either workload=... or op=..., or use " +
-                    "a default driver (driver=___). This includes " +
-                    ServiceLoader.load(DriverAdapter.class).stream()
-                            .filter(p -> {
-                                AnnotatedType[] annotatedInterfaces = p.type().getAnnotatedInterfaces();
-                                for (AnnotatedType ai : annotatedInterfaces) {
-                                    if (ai.getType().equals(SyntheticOpTemplateProvider.class)) {
-                                        return true;
-                                    }
-                                }
-                                return false;
-                            })
-                            .map(d -> d.get().getAdapterName())
-                            .collect(Collectors.joining(",")));
         }
+//        if (filteredOps.isEmpty()) {
+//            throw new BasicError("There were no active op templates with tag filter '" + tagfilter + '\'');
+//        }
 
+//        if (filteredOps.isEmpty()) {
+//            throw new OpConfigError("No op templates found. You must provide either workload=... or op=..., or use " +
+//                "a default driver (driver=___). This includes " +
+//                ServiceLoader.load(DriverAdapter.class).stream()
+//                    .filter(p -> {
+//                        AnnotatedType[] annotatedInterfaces = p.type().getAnnotatedInterfaces();
+//                        for (AnnotatedType ai : annotatedInterfaces) {
+//                            if (ai.getType().equals(SyntheticOpTemplateProvider.class)) {
+//                                return true;
+//                            }
+//                        }
+//                        return false;
+//                    })
+//                    .map(d -> d.get().getAdapterName())
+//                    .collect(Collectors.joining(",")));
+//        }
+//
         return filteredOps;
     }
 
@@ -554,18 +559,17 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
      * @param opinit
      *     A function to map an OpTemplate to the executable operation form required by
      *     the native driver for this activity.
-     * @param defaultAdapter
+     * @param defaultAdapter The adapter which will be used for any op templates with no explicit adapter
      * @return The sequence of operations as determined by filtering and ratios
      */
     @Deprecated(forRemoval = true)
-    protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit, boolean strict, Optional<DriverAdapter<?,?>> defaultAdapter) {
+    protected <O> OpSequence<OpDispenser<? extends O>> createOpSequence(Function<OpTemplate, OpDispenser<? extends O>> opinit, boolean strict, DriverAdapter<?, ?> defaultAdapter) {
 
         var stmts = loadOpTemplates(defaultAdapter);
 
         List<Long> ratios = new ArrayList<>(stmts.size());
 
-        for (int i = 0; i < stmts.size(); i++) {
+        for (OpTemplate opTemplate : stmts) {
-            OpTemplate opTemplate = stmts.get(i);
             long ratio = opTemplate.removeParamOrDefault("ratio", 1);
             ratios.add(ratio);
         }
@@ -599,29 +603,37 @@ public class SimpleActivity extends NBBaseComponent implements Activity {
             String op = activityDef.getParams().getOptionalString("op").orElse(null);
             String stmt = activityDef.getParams().getOptionalString("stmt", "statement").orElse(null);
             String workload = activityDef.getParams().getOptionalString("workload").orElse(null);
-            if ((op!=null ? 1 : 0) + (stmt!=null ? 1 : 0) + (workload!=null ? 1 : 0) > 1) {
+
+            if ((op != null ? 1 : 0) + (stmt != null ? 1 : 0) + (workload != null ? 1 : 0) > 1) {
                 throw new OpConfigError("Only op, statement, or workload may be provided, not more than one.");
             }
 
-            if (op!=null && op.matches("^\\{[^}]+:[^}]+}$(?s)(?m)")) {
+
-                workloadSource = "commandline: (op/json): '" + op + "'";
+            if (workload != null && OpsLoader.isJson(workload)) {
-                return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
+                workloadSource = "commandline: (workload/json):" + workload;
-            } else if (op!=null && op.matches("^\\[[^]]+]$")) {
+                return OpsLoader.loadString(workload, OpTemplateFormat.json, activityDef.getParams(), null);
-                workloadSource = "commandline: (op/json): '" + op + "'";
+            } else if (workload != null && OpsLoader.isYaml(workload)) {
-                return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
+                workloadSource = "commandline: (workload/yaml):" + workload;
-            } else if (op!=null) {
+                return OpsLoader.loadString(workload, OpTemplateFormat.yaml, activityDef.getParams(), null);
-                workloadSource = "commandline: (op/inline): '" + op + "'";
+            } else if (workload != null) {
-                return OpsLoader.loadString(op, OpTemplateFormat.inline, activityDef.getParams(), null);
+                return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
-            } else if (stmt!=null) {
+            }
+
+            if (stmt != null) {
                 workloadSource = "commandline: (stmt/inline): '" + stmt + "'";
                 return OpsLoader.loadString(stmt, OpTemplateFormat.inline, activityDef.getParams(), null);
-            } else if (workload!=null) {
-                workloadSource = "yaml:" + workload;
-                return OpsLoader.loadPath(workload, activityDef.getParams(), "activities");
-            } else {
-                return OpsDocList.none();
             }
 
+            if (op != null && OpsLoader.isJson(op)) {
+                workloadSource = "commandline: (op/json): '" + op + "'";
+                return OpsLoader.loadString(op, OpTemplateFormat.json, activityDef.getParams(), null);
+            }
+            else if (op != null) {
+                workloadSource = "commandline: (op/inline): '" + op + "'";
+                return OpsLoader.loadString(op, OpTemplateFormat.inline, activityDef.getParams(), null);
+            }
+            return OpsDocList.none();
+
         } catch (Exception e) {
             throw new OpConfigError("Error loading op templates: " + e, workloadSource, e);
         }