IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

ModuloCSVLineToString has more robust errors

Browse Source
This commit is contained in:
Jonathan Shook
2020-03-26 16:02:22 -05:00
parent b1bb42aa41
commit b487a7f936
2 changed files with 16 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/long_string/ModuloCSVLineToString.java
View File

@@ -1,69 +0,0 @@
/*
*
* Copyright 2015 Jonathan Shook
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.activitytype.cql.datamappers.functions.long_string;
import io.nosqlbench.virtdata.annotations.Example;
import io.nosqlbench.virtdata.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.api.VirtDataResources;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.ModuloLineToString;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
/**
* Select a value from a CSV file line by modulo division against the number
* of lines in the file. The second parameter is the field name, and this must
* be provided in the CSV header line as written.
*/
@ThreadSafeMapper
public class ModuloCSVLineToString implements LongFunction<String> {
private final static Logger logger = LoggerFactory.getLogger(ModuloLineToString.class);
private List<String> lines = new ArrayList<>();
private String filename;
@Example({"ModuloCSVLineToString('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToString(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = VirtDataResources.readFileCSV(filename);
int column = csvp.getHeaderMap().get(fieldname);
for (CSVRecord strings : csvp) {
lines.add(strings.get(column));
}
}
@Override
public String apply(long input) {
int itemIdx = (int) (input % lines.size()) % Integer.MAX_VALUE;
String item = lines.get(itemIdx);
return item;
}
public String toString() {
return getClass().getSimpleName() + ":" + filename;
}
}

18
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloCSVLineToString.java
View File

@@ -26,8 +26,10 @@ import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import java.security.InvalidParameterException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.function.LongFunction; import java.util.function.LongFunction;
/** /**
@@ -37,7 +39,9 @@ import java.util.function.LongFunction;
*/ */
@ThreadSafeMapper @ThreadSafeMapper
public class ModuloCSVLineToString implements LongFunction<String> { public class ModuloCSVLineToString implements LongFunction<String> {
private final static Logger logger = LogManager.getLogger(ModuloLineToString.class);private List<String> lines = new ArrayList<>(); private final static Logger logger = LogManager.getLogger(ModuloLineToString.class);
private List<String> lines = new ArrayList<>();
private String filename; private String filename;
@@ -45,7 +49,17 @@ public class ModuloCSVLineToString implements LongFunction<String> {
public ModuloCSVLineToString(String filename, String fieldname) { public ModuloCSVLineToString(String filename, String fieldname) {
this.filename = filename; this.filename = filename;
CSVParser csvp = VirtDataResources.readFileCSV(filename); CSVParser csvp = VirtDataResources.readFileCSV(filename);
int column = csvp.getHeaderMap().get(fieldname); Map<String, Integer> headerMap = csvp.getHeaderMap();
if (headerMap==null || headerMap.isEmpty()) {
throw new RuntimeException("There were not headers for file "+ filename + ". " + ModuloCSVLineToString.class.getSimpleName() + " requires headers.");
}
Integer column = headerMap.get(fieldname);
if (column==null) {
throw new RuntimeException("Could not find the named column header '" + fieldname + "' in file " + filename);
}
for (CSVRecord strings : csvp) { for (CSVRecord strings : csvp) {
lines.add(strings.get(column)); lines.add(strings.get(column));
} }