remove legacy driver-stdout and move stdout.md into stdout adapter

This commit is contained in:
Jonathan Shook
2022-06-23 00:41:56 -05:00
parent af081ff56f
commit afebe1b30e
15 changed files with 0 additions and 689 deletions

View File

@@ -1,48 +0,0 @@
<!--
~ Copyright (c) 2022 nosqlbench
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>driver-stdout</artifactId>
<packaging>jar</packaging>
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.15-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
<name>${project.artifactId}</name>
<description>
An nosqlbench ActivityType (AT) driver module;
Provides basic formatting and output to stdout or files.
</description>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.15-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
</project>

View File

@@ -1,79 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import com.codahale.metrics.Timer;
import io.nosqlbench.engine.api.activityapi.core.BaseAsyncAction;
import io.nosqlbench.engine.api.activityapi.core.ops.fluent.opfacets.StartedOp;
import io.nosqlbench.engine.api.activityapi.core.ops.fluent.opfacets.TrackedOp;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.function.LongFunction;
@SuppressWarnings("Duplicates")
public class AsyncStdoutAction extends BaseAsyncAction<StdoutOpContext, StdoutActivity> {
private final static Logger logger = LogManager.getLogger(AsyncStdoutAction.class);
private OpSequence<StringBindings> sequencer;
public AsyncStdoutAction(int slot, StdoutActivity activity) {
super(activity, slot);
}
@Override
public void onActivityDefUpdate(ActivityDef activityDef) {
super.onActivityDefUpdate(activityDef);
this.sequencer = activity.getOpSequence();
}
public StdoutOpContext allocateOpData(long cycle) {
StdoutOpContext opc = new StdoutOpContext();
try (Timer.Context bindTime = activity.bindTimer.time()) {
opc.stringBindings = sequencer.apply(cycle);
opc.statement = opc.stringBindings.bind(cycle);
if (activity.getShowstmts()) {
logger.info("STMT(cycle=" + cycle + "):\n" + opc.statement);
}
}
return opc;
}
@Override
public void startOpCycle(TrackedOp<StdoutOpContext> opc) {
StartedOp<StdoutOpContext> started = opc.start();
int result=0;
try (Timer.Context executeTime = activity.executeTimer.time()) {
activity.write(opc.getOpData().statement);
} catch (Exception e) {
result=1;
started.fail(result);
throw new RuntimeException("Error writing output:" + e, e);
} finally {
started.succeed(result);
}
}
@Override
public LongFunction<StdoutOpContext> getOpInitFunction() {
return this::allocateOpData;
}
}

View File

@@ -1,67 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import com.codahale.metrics.Timer;
import io.nosqlbench.engine.api.activityapi.core.SyncAction;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@SuppressWarnings("Duplicates")
public class StdoutAction implements SyncAction {
private static final Logger logger = LogManager.getLogger(StdoutAction.class);
private final int slot;
private final StdoutActivity activity;
private final int maxTries = 10;
private boolean showstmts;
private OpSequence<StringBindings> opsource;
public StdoutAction(int slot, StdoutActivity activity) {
this.slot = slot;
this.activity = activity;
}
@Override
public void init() {
this.opsource = activity.getOpSequence();
}
@Override
public int runCycle(long cycle) {
StringBindings stringBindings;
String statement = null;
try (Timer.Context bindTime = activity.bindTimer.time()) {
stringBindings = opsource.apply(cycle);
statement = stringBindings.bind(cycle);
showstmts = activity.getShowstmts();
if (showstmts) {
logger.info("STMT(cycle=" + cycle + "):\n" + statement);
}
}
try (Timer.Context executeTime = activity.executeTimer.time()) {
activity.write(statement);
} catch (Exception e) {
throw new RuntimeException("Error writing output:" + e, e);
}
return 0;
}
}

View File

@@ -1,247 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import io.nosqlbench.adapters.stdout.TemplateFormat;
import io.nosqlbench.engine.api.activityapi.core.ActivityDefObserver;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityapi.planning.SequencePlanner;
import io.nosqlbench.engine.api.activityapi.planning.SequencerType;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.engine.api.activityimpl.ParameterMap;
import io.nosqlbench.engine.api.activityimpl.SimpleActivity;
import io.nosqlbench.engine.api.metrics.ActivityMetrics;
import io.nosqlbench.engine.api.metrics.ExceptionMeterMetrics;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplate;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Pattern;
@SuppressWarnings("Duplicates")
public class StdoutActivity extends SimpleActivity implements ActivityDefObserver {
private final static Logger logger = LogManager.getLogger(StdoutActivity.class);
private final Boolean showstmts;
private final StmtsDocList stmtsDocList;
public Timer bindTimer;
public Timer executeTimer;
public Timer resultTimer;
public Histogram triesHisto;
private Writer pw;
private final String fileName;
private ExceptionMeterMetrics exceptionMeterMetrics;
private int retry_delay = 0;
private int retries;
public OpSequence<StringBindings> getOpSequence() {
return opSequence;
}
private OpSequence<StringBindings> opSequence;
public StdoutActivity(ActivityDef activityDef) {
super(activityDef);
String yaml_loc = activityDef.getParams().getOptionalString("yaml", "workload").orElse("default");
this.showstmts = activityDef.getParams().getOptionalBoolean("showstatements").orElse(false);
this.fileName = activityDef.getParams().getOptionalString("filename").orElse("stdout");
this.stmtsDocList = StatementsLoader.loadPath(logger, yaml_loc, activityDef.getParams(), "activities");
}
@Override
public void shutdownActivity() {
try {
if (pw != null) {
if (!fileName.equalsIgnoreCase("stdout")) {
logger.trace("Closing non-stdout output stream.");
pw.close();
}
}
} catch (Exception e) {
logger.warn("error closing writer:" + e, e);
}
}
@Override
public void initActivity() {
logger.debug("initializing activity: " + this.activityDef.getAlias());
exceptionMeterMetrics = new ExceptionMeterMetrics(activityDef);
onActivityDefUpdate(activityDef);
opSequence = initOpSequencer();
setDefaultsFromOpSequence(opSequence);
bindTimer = ActivityMetrics.timer(activityDef, "bind", this.getHdrDigits());
executeTimer = ActivityMetrics.timer(activityDef, "execute", this.getHdrDigits());
resultTimer = ActivityMetrics.timer(activityDef, "result", this.getHdrDigits());
triesHisto = ActivityMetrics.histogram(activityDef, "tries", this.getHdrDigits());
this.pw = createPrintWriter();
}
protected Writer createPrintWriter() {
PrintWriter pw;
if (fileName.equalsIgnoreCase("stdout")) {
pw = getConsoleOut();
} else {
try {
pw = new PrintWriter(fileName);
pw.print("");
} catch (FileNotFoundException e) {
e.printStackTrace();
throw new RuntimeException("Error initializing printwriter:" + e, e);
}
}
return pw;
}
private OpSequence<StringBindings> initOpSequencer() {
//List<StringBindingsTemplate> stringBindingsTemplates = new ArrayList<>();
SequencerType sequencerType = SequencerType.valueOf(
getParams().getOptionalString("seq").orElse("bucket")
);
SequencePlanner<StringBindings> sequencer = new SequencePlanner<>(sequencerType);
String tagfilter = activityDef.getParams().getOptionalString("tags").orElse("");
List<OpTemplate> stmts = stmtsDocList.getStmts(tagfilter);
String format = getParams().getOptionalString("format").orElse(null);
if ((stmts.size() == 0 && stmtsDocList.getDocBindings().size() > 0) || format != null) {
if (format != null && format.startsWith("diag")) {
logger.info("Creating diagnostic log for resolver construction...");
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
String diagnostics = bt.getDiagnostics();
getConsoleOut().println(diagnostics);
getConsoleOut().flush();
this.getActivityController().stopActivityWithReasonAsync("diagnostics created for stdout");
} else {
logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
Set<String> activeBindingNames = new LinkedHashSet<>();
String bindings = getActivityDef().getParams().getOptionalString("bindings").orElse("doc");
activeBindingNames.addAll(stmtsDocList.getDocBindings().keySet());
Pattern bindingsFilter = Pattern.compile(bindings.equalsIgnoreCase("doc") ? ".*" : bindings);
Set<String> filteredBindingNames = new LinkedHashSet<>();
activeBindingNames
.stream()
.filter(n -> {
if (bindingsFilter.matcher(n).matches()) {
logger.trace("bindings filter kept binding '" + n + "'");
return true;
} else {
logger.trace("bindings filter removed binding '" + n + "'");
return false;
}
})
.forEach(filteredBindingNames::add);
activeBindingNames = filteredBindingNames;
String generatedStmt = genStatementTemplate(activeBindingNames);
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
StringBindings sb = new StringBindings(generatedStmt,bt.getMap());
sequencer.addOp(sb, 1L);
}
} else if (stmts.size() > 0) {
for (OpTemplate stmt : stmts) {
ParsedTemplate parsed = stmt.getParsed().orElseThrow();
BindingsTemplate bt = new BindingsTemplate(parsed.getBindPoints());
String statement = parsed.getPositionalStatement(Function.identity());
Objects.requireNonNull(statement);
if (!statement.endsWith("\n") && getParams().getOptionalBoolean("newline").orElse(true)) {
statement = statement + "\n";
}
StringBindingsTemplate sbt = new StringBindingsTemplate(stmt.getStmt().orElseThrow(), bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb, stmt.getParamOrDefault("ratio", 1));
}
} else {
logger.error("Unable to create a stdout statement if you have no active statements or bindings configured.");
}
OpSequence<StringBindings> opSequence = sequencer.resolve();
return opSequence;
}
private String genStatementTemplate(Set<String> keySet) {
TemplateFormat format = getParams().getOptionalString("format")
.map(TemplateFormat::valueOf)
.orElse(TemplateFormat.assignments);
boolean ensureNewline = getParams().getOptionalBoolean("newline")
.orElse(true);
String stmtTemplate = format.format(ensureNewline, new ArrayList<>(keySet));
return stmtTemplate;
}
@Override
public void onActivityDefUpdate(ActivityDef activityDef) {
super.onActivityDefUpdate(activityDef);
ParameterMap params = activityDef.getParams();
this.retry_delay = params.getOptionalInteger("retry_delay").orElse(1000);
this.retries = params.getOptionalInteger("retries").orElse(3);
}
public synchronized void write(String statement) {
int tries = 0;
Exception e = null;
while (tries < retries) {
tries++;
if (pw == null) {
pw = createPrintWriter();
}
try {
pw.write(statement);
pw.flush();
return;
} catch (Exception error) {
logger.warn("Error during write:" + error, error);
if (retry_delay > 0) {
try {
Thread.sleep(retry_delay);
} catch (InterruptedException ignored) {
}
}
}
}
throw new RuntimeException("Retries exhausted: " + tries + "/" + retries);
}
public Boolean getShowstmts() {
return showstmts;
}
}

View File

@@ -1,62 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import io.nosqlbench.engine.api.activityapi.core.Action;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityapi.core.ActivityType;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import java.util.Optional;
public class StdoutActivityType implements ActivityType<StdoutActivity> {
@Override
public StdoutActivity getActivity(ActivityDef activityDef) {
// sanity check that we have a yaml parameter, which contains our statements and bindings
Optional<String> stmtsrc = activityDef.getParams().getOptionalString("op", "stmt", "statement", "yaml", "workload");
if (stmtsrc.isEmpty()) {
throw new RuntimeException("Without a workload or op parameter, there is nothing to do. (Add a workload (yaml file) or an op= template, like" +
" op='cycle={{Identity()}}'");
}
return new StdoutActivity(activityDef);
}
@Override
public ActionDispenser getActionDispenser(StdoutActivity activity) {
return new StdoutActionDispenser(activity);
}
private static class StdoutActionDispenser implements ActionDispenser {
private final StdoutActivity activity;
private StdoutActionDispenser(StdoutActivity activity) {
this.activity = activity;
}
@Override
public Action getAction(int slot) {
if (activity.getActivityDef().getParams().getOptionalString("async").isPresent()) {
return new AsyncStdoutAction(slot, activity);
}
return new StdoutAction(slot, activity);
}
}
}

View File

@@ -1,24 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import io.nosqlbench.virtdata.core.templates.StringBindings;
public class StdoutOpContext {
StringBindings stringBindings;
String statement;
}

View File

@@ -1,35 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapters.stdout;
import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.RunnableOp;
public class StdoutOp implements RunnableOp {
private final StdoutSpace ctx;
private final String text;
public StdoutOp(StdoutSpace ctx, String text) {
this.ctx = ctx;
this.text = text;
}
@Override
public void run() {
ctx.write(text);
}
}

View File

@@ -1,2 +0,0 @@
bindings:
line: DirectoryLines(<<datafile:none-specified>>,<<datafile_pattern:none-specified>>)

View File

@@ -1,11 +0,0 @@
tags:
type: testtag
kind: somekind
oevure: bananas
name: outerblock
bindings:
cycle: Identity()
thread: ThreadNumToLong()
bar: NumberNameToString()
foo: NumberNameToString()
customer: NumberNameToString()

View File

@@ -1,4 +0,0 @@
bindings:
alpha: Identity()
beta: NumberNameToString()
gamma: Combinations('0-9A-F;0-9;A-Z;_;p;r;o;')

View File

@@ -1,2 +0,0 @@
# stdout help topics
- stdout

View File

@@ -1,58 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import io.nosqlbench.adapters.stdout.TemplateFormat;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import static org.assertj.core.api.Assertions.assertThat;
public class StatementFormattersTest {
@Test
public void testCsvFormats() {
String csv = TemplateFormat.csv.format(false,Arrays.asList("alpha", "beta"));
assertThat(csv).isEqualTo("{alpha},{beta}");
}
@Test
public void testInlineJSONFormat() {
String csv = TemplateFormat.inlinejson.format(false,Arrays.asList("alpha", "beta"));
assertThat(csv).isEqualTo("{\"alpha\":\"{alpha}\", \"beta\":\"{beta}\"}");
}
@Test
public void testBlockJSONFormat() {
String csv = TemplateFormat.json.format(false,Arrays.asList("alpha", "beta"));
assertThat(csv).isEqualTo("{\n \"alpha\":\"{alpha}\",\n \"beta\":\"{beta}\"\n}");
}
@Test
public void testAssignmentsFormat() {
String csv = TemplateFormat.assignments.format(false,Arrays.asList("alpha", "beta"));
assertThat(csv).isEqualTo("alpha={alpha} beta={beta}");
}
@Test
public void testReadoutFormat() {
String csv = TemplateFormat.readout.format(false,Arrays.asList("alpha", "beta"));
assertThat(csv).isEqualTo("alpha : {alpha}\n beta : {beta}");
}
}

View File

@@ -1,37 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.activitytype.stdout;
import io.nosqlbench.engine.api.activityapi.core.Action;
import io.nosqlbench.engine.api.activityapi.core.ActionDispenser;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import org.junit.jupiter.api.Test;
/**
* Created by sebastianestevez on 5/5/17.
*/
public class StdoutActivityTypeTest {
@Test
public void testDiagActivity() {
StdoutActivityType stdoutAt = new StdoutActivityType();
ActivityDef ad = ActivityDef.parseActivityDef("driver=stdout; yaml=stdout-test;");
StdoutActivity stdoutActivity = stdoutAt.getActivity(ad);
ActionDispenser actionDispenser = stdoutAt.getActionDispenser(stdoutActivity);
Action action = actionDispenser.getAction(1);
}
}

View File

@@ -1,13 +0,0 @@
tags:
type: testtag
kind: somekind
oevure: bananas
name: outerblock
statements:
- foo
- bar
- customer
bindings:
bar: NumberNameToString()
foo: NumberNameToString()
customer: NumberNameToString()