diff --git a/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutput.java b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutput.java new file mode 100644 index 000000000..0de925ab6 --- /dev/null +++ b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutput.java @@ -0,0 +1,4 @@ +package io.nosqlbench.engine.extensions.csvoutput; + +public class CsvOutput { +} diff --git a/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputPluginData.java b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputPluginData.java new file mode 100644 index 000000000..f67e054de --- /dev/null +++ b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputPluginData.java @@ -0,0 +1,22 @@ +package io.nosqlbench.engine.extensions.csvoutput; + +import com.codahale.metrics.MetricRegistry; +import io.nosqlbench.engine.api.extensions.ScriptingPluginInfo; +import io.nosqlbench.nb.annotations.Service; +import org.apache.logging.log4j.Logger; + +import javax.script.ScriptContext; + +@Service(value = ScriptingPluginInfo.class,selector = "csvoutput") +public class CsvOutputPluginData implements ScriptingPluginInfo { + + @Override + public String getDescription() { + return "Write CSV output to a named file"; + } + + @Override + public CsvOutputPluginInstance getExtensionObject(Logger logger, MetricRegistry metricRegistry, ScriptContext scriptContext) { + return new CsvOutputPluginInstance(); + } +} diff --git a/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputPluginInstance.java b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputPluginInstance.java new file mode 100644 index 000000000..0158001e3 --- /dev/null +++ b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputPluginInstance.java @@ -0,0 +1,8 @@ +package io.nosqlbench.engine.extensions.csvoutput; + +public class CsvOutputPluginInstance { + + public CsvOutput open(String filename, String... headers) { + return new CsvOutputWriter(filename, headers); + } +} diff --git a/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputWriter.java b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputWriter.java new file mode 100644 index 000000000..2dfcd6ca7 --- /dev/null +++ b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputWriter.java @@ -0,0 +1,75 @@ +package io.nosqlbench.engine.extensions.csvoutput; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.graalvm.polyglot.Value; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +public class CsvOutputWriter extends CsvOutput { + + private final CSVPrinter printer; + private final FileWriter filewriter; + private final LinkedHashSet headerKeys; + private final String filename; + + public CsvOutputWriter(String filename, String... headers) { + this.filename = filename; + CSVFormat fmt = CSVFormat.DEFAULT; + this.headerKeys = new LinkedHashSet<>(Arrays.asList(headers)); + try { + this.filewriter = new FileWriter(filename); + this.printer = new CSVPrinter(filewriter,fmt); + if (Files.size(Path.of(filename))==0) { + printer.printRecord(headerKeys); + printer.flush(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public CsvOutputWriter write(Value value) { + List lineout = new ArrayList<>(); + Map provided = new HashMap<>(); + if (value.isHostObject()) { + Object o = value.asHostObject(); + if (o instanceof Map) { + ((Map) o).forEach((k,v) -> { + provided.put(k.toString(),v.toString()); + }); + } else { + throw new RuntimeException("host object provided as '" + o.getClass().getCanonicalName()+ ", but only Maps are supported."); + } + } else if (value.hasMembers()) { + for (String vkey : value.getMemberKeys()) { + provided.put(vkey,value.getMember(vkey).toString()); + } + } else { + throw new RuntimeException("Value was not a Map host object nor a type with members."); + } + + for (String headerKey : headerKeys) { + if (provided.containsKey(headerKey)) { + lineout.add(provided.remove(headerKey)); + } else { + lineout.add(""); + } + } + if (provided.size()>0) { + throw new RuntimeException("Unqualified column was emitted for file '" + filename); + } + + try { + printer.printRecord(lineout); + printer.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } +} diff --git a/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/csvoutput.md b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/csvoutput.md new file mode 100644 index 000000000..502fbd46a --- /dev/null +++ b/engine-extensions/src/main/java/io/nosqlbench/engine/extensions/csvoutput/csvoutput.md @@ -0,0 +1,13 @@ +csvoutput extension +=================== + +This extension makes it easy to start writing CSV data to a file, +using a defined set of headers. + +### Examples + +Open a writer and write a row: + + var out=csvoutput.open('output.csv','time','value'); + out.write({'time':23,'value':23}); + diff --git a/engine-extensions/src/test/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputWriterTest.java b/engine-extensions/src/test/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputWriterTest.java new file mode 100644 index 000000000..a91daeb76 --- /dev/null +++ b/engine-extensions/src/test/java/io/nosqlbench/engine/extensions/csvoutput/CsvOutputWriterTest.java @@ -0,0 +1,22 @@ +package io.nosqlbench.engine.extensions.csvoutput; + +import org.assertj.core.util.Files; +import org.graalvm.polyglot.Value; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.Map; + +public class CsvOutputWriterTest { + + @Test + public void testCsvOutputWriter() { + File tmpfile = Files.newTemporaryFile(); + tmpfile.deleteOnExit(); + System.out.println("tmpfile="+ tmpfile.getPath()); + CsvOutputWriter out = new CsvOutputWriter(tmpfile.getPath(), "one", "two"); + out.write(Value.asValue(Map.of("one","one_","two","two_"))); + } + + +} diff --git a/nb/src/test/java/io/nosqlbench/engine/core/script/AsyncScriptIntegrationTests.java b/nb/src/test/java/io/nosqlbench/engine/core/script/AsyncScriptIntegrationTests.java index cbfde08a4..1c3e7c3a8 100644 --- a/nb/src/test/java/io/nosqlbench/engine/core/script/AsyncScriptIntegrationTests.java +++ b/nb/src/test/java/io/nosqlbench/engine/core/script/AsyncScriptIntegrationTests.java @@ -178,6 +178,16 @@ public class AsyncScriptIntegrationTests { assertThat(logdata.split("Tag=testhistostatslogger.cycles.servicetime,").length).isGreaterThanOrEqualTo(2); } + @Test + public void testExtensionCsvOutput() throws IOException { + ScenarioResult scenarioResult = runScenario("extension_csvoutput"); + List strings = Files.readAllLines(Paths.get( + "logs/csvoutputtestfile.csv")); + String logdata = strings.stream().collect(Collectors.joining("\n")); + assertThat(logdata).contains("header1,header2"); + assertThat(logdata).contains("value1,value2"); + } + @Test public void testExtensionHistogramLogger() throws IOException { ScenarioResult scenarioResult = runScenario("extension_histologger"); diff --git a/nb/src/test/resources/scripts/async/extension_csvoutput.js b/nb/src/test/resources/scripts/async/extension_csvoutput.js new file mode 100644 index 000000000..77cfe161c --- /dev/null +++ b/nb/src/test/resources/scripts/async/extension_csvoutput.js @@ -0,0 +1,20 @@ +/* + * + * Copyright 2016 jshook + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +var csvlogger = csvoutput.open("logs/csvoutputtestfile.csv","header1","header2"); + +csvlogger.write({"header1": "value1","header2":"value2"});