IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

merge fixups, NBIO refactorings, UI stubs

Browse Source
This commit is contained in:
Jonathan Shook
2020-04-09 12:44:31 -05:00
parent 7fa5ec9ebd 510a9dd354
commit 5e8c5bdfca
83 changed files with 1269 additions and 817 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java
View File

@@ -1,21 +1,60 @@
package io.nosqlbench.nb.api.content;
import java.io.*;
import java.net.URI;
import java.nio.CharBuffer;
import java.nio.file.FileSystem;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.spi.FileSystemProvider;
import java.util.function.Supplier;
/**
* A generic content wrapper for anything that can be given to a NoSQLBench runtime
* using a specific type of locator.
*
* @param <T>
*/
public interface Content<T> {
public interface Content<T> extends Supplier<CharSequence>, Comparable<Content<?>> {
T getLocation();
URI getURI();
CharBuffer getCharBuffer();
Path asPath();
public default String asString() {
return getCharBuffer().toString();
}
Path asPath();
CharBuffer getCharBuffer();
@Override
default CharSequence get() {
return getCharBuffer();
}
default int compareTo(Content<?> other) {
return getURI().compareTo(other.getURI());
}
default Reader getReader() {
InputStream inputStream = getInputStream();
return new InputStreamReader(inputStream);
}
default InputStream getInputStream() {
try {
Path path = asPath();
FileSystem fileSystem = path.getFileSystem();
FileSystemProvider provider = fileSystem.provider();
InputStream stream = provider.newInputStream(path, StandardOpenOption.READ);
return stream;
} catch (IOException ignored) {
}
String stringdata = getCharBuffer().toString();
return new ByteArrayInputStream(stringdata.getBytes());
}
}

11
nb-api/src/main/java/io/nosqlbench/nb/api/content/ContentResolver.java
View File

@@ -2,6 +2,7 @@ package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
public interface ContentResolver {
@@ -25,15 +26,13 @@ public interface ContentResolver {
* @param uri The URI of a content location, like a file name or URL.
* @return A content element which may then be used to access the content
*/
Content<?> resolve(URI uri);
default Content<?> resolve(String uri) {
List<Content<?>> resolve(URI uri);
default List<Content<?>> resolve(String uri) {
return resolve(URI.create(uri));
}
Optional<Path> resolveDirectory(URI uri);
default Optional<Path> resolveDirectory(String uri) {
List<Path> resolveDirectory(URI uri);
default List<Path> resolveDirectory(String uri) {
return resolveDirectory(URI.create(uri));
}

376
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java
View File

@@ -1,9 +1,21 @@
package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.content.fluent.NBPathsAPI;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.nio.CharBuffer;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* NBIO is a helper utility packaged as a search builder and fluent API.
@@ -19,100 +31,125 @@ import java.util.*;
public class NBIO implements NBPathsAPI.Facets {
private URIResolver resolver;
private MatchType matchas = MatchType.exact;
private List<String> names = new ArrayList<>();
private List<String> extensions = new ArrayList<>();
private List<String> searchPaths = new ArrayList<>();
private enum MatchType {
exact,
suffix,
pattern
}
private List<String> prefixes = new ArrayList<>();
private NBIO() {
}
private NBIO(URIResolver resolver,
MatchType matchas,
List<String> searchPaths,
List<String> prefixes,
List<String> names,
List<String> extensions) {
this.resolver = resolver;
this.matchas = matchas;
this.searchPaths = searchPaths;
this.prefixes = prefixes;
this.names = names;
this.extensions = extensions;
}
public static List<String> readLines(String filename) {
Content<?> data = NBIO.all().prefix("data").name(filename).one();
String[] split = data.getCharBuffer().toString().split("\n");
return Arrays.asList(split);
}
public static CSVParser readFileCSV(String filename, String... searchPaths) {
return NBIO.readFileDelimCSV(filename, ',', searchPaths);
}
public static CSVParser readFileDelimCSV(String filename,char delim, String... searchPaths) {
Reader reader = NBIO.readReader(filename, searchPaths);
CSVFormat format = CSVFormat.newFormat(delim).withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static InputStream readInputStream(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getInputStream();
}
private static Reader readReader(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getReader();
}
public static CharBuffer readCharBuffer(String fileName, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(fileName).one().getCharBuffer();
}
public static Path getFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
return first.orElseThrow().asPath();
}
public static Optional<Path> findFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
Optional<Path> path = first.map(Content::asPath);
return path;
}
public static InputStream readInputStream(String fromPath, String yaml, String[] searchPaths) {
return null;
}
@Override
public NBPathsAPI.ForContentSource localContent() {
public NBPathsAPI.GetPrefix localContent() {
this.resolver = URIResolvers.inFS().inCP();
return this;
}
@Override
public NBPathsAPI.ForContentSource remoteContent() {
public NBPathsAPI.GetPrefix remoteContent() {
this.resolver = URIResolvers.inURLs();
return this;
}
@Override
public NBPathsAPI.ForContentSource internalContent() {
public NBPathsAPI.GetPrefix internalContent() {
this.resolver = URIResolvers.inClasspath();
return this;
}
@Override
public NBPathsAPI.ForContentSource fileContent() {
public NBPathsAPI.GetPrefix fileContent() {
this.resolver = URIResolvers.inFS();
return this;
}
@Override
public NBPathsAPI.ForContentSource allContent() {
public NBPathsAPI.GetPrefix allContent() {
this.resolver = URIResolvers.inFS().inCP().inURLs();
return this;
}
@Override
public NBPathsAPI.WantsContentName exact() {
return new NBIO(resolver, MatchType.exact, searchPaths, names, extensions);
}
@Override
public NBPathsAPI.WantsContentName matchtail() {
return new NBIO(resolver, MatchType.suffix, searchPaths, names, extensions);
}
@Override
public NBPathsAPI.WantsContentName regex() {
return new NBIO(resolver, MatchType.pattern, searchPaths, names, extensions);
}
@Override
public NBPathsAPI.ForPrefix prefix(String... searchPaths) {
ArrayList<String> addingPaths = new ArrayList<>(this.searchPaths);
public NBPathsAPI.GetPrefix prefix(String... searchPaths) {
ArrayList<String> addingPaths = new ArrayList<>(this.prefixes);
addingPaths.addAll(Arrays.asList(searchPaths));
return new NBIO(resolver, matchas, addingPaths, names, extensions);
return new NBIO(resolver, addingPaths, names, extensions);
}
@Override
public NBPathsAPI.ForName name(String... searchNames) {
public NBPathsAPI.GetExtension name(String... searchNames) {
ArrayList<String> addingNames = new ArrayList<>(this.names);
addingNames.addAll(Arrays.asList(searchNames));
return new NBIO(resolver, matchas, searchPaths, addingNames, extensions);
return new NBIO(resolver, prefixes, addingNames, extensions);
}
@Override
public NBPathsAPI.ForExtension extension(String... extensions) {
public NBPathsAPI.DoSearch extension(String... extensions) {
ArrayList<String> addingExtensions = new ArrayList<>(this.extensions);
for (String addingExtension : extensions) {
addingExtensions.add(addingExtension.startsWith(".") ? addingExtension : "." + addingExtension);
addingExtensions.add(dotExtension(addingExtension));
}
return new NBIO(resolver, matchas, searchPaths, names, addingExtensions);
return new NBIO(resolver, prefixes, names, addingExtensions);
}
/**
@@ -120,7 +157,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource all() {
public static NBPathsAPI.GetPrefix all() {
return new NBIO().allContent();
}
@@ -129,7 +166,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource classpath() {
public static NBPathsAPI.GetPrefix classpath() {
return new NBIO().internalContent();
}
@@ -138,7 +175,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource fs() {
public static NBPathsAPI.GetPrefix fs() {
return new NBIO().fileContent();
}
@@ -147,7 +184,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource local() {
public static NBPathsAPI.GetPrefix local() {
return new NBIO().localContent();
}
@@ -156,74 +193,243 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource remote() {
public static NBPathsAPI.GetPrefix remote() {
return new NBIO().remoteContent();
}
@Override
public Optional<Content<?>> first() {
Content<?> found = null;
LinkedHashSet<String> specificPathsToSearch = expandSearches();
for (String candidatePath : specificPathsToSearch) {
Content<?> content = resolver.resolve(candidatePath);
if (content!=null) {
return Optional.of(content);
}
List<Content<?>> list = list();
if (list.size() > 0) {
return Optional.of(list.get(0));
} else {
return Optional.empty();
}
return Optional.empty();
}
public Optional<Content<?>> maybeOne() {
List<Content<?>> list = list();
if (list.size() > 1) {
throw new BasicError("Found more than one source for " + this.toString() + ", but expected to find one at" +
" most.");
}
throw new RuntimeException("Invalid code, go fix it, this should never happen.");
}
@Override
public List<Optional<Content<?>>> resolveEach() {
List<Optional<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandSearches(name);
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
content = resolver.resolve(slotSearchPath);
if (content!=null) {
break;
}
}
resolved.add(Optional.ofNullable(content));
public Content<?> one() {
List<Content<?>> list = list();
if (list.size() == 0) {
throw new BasicError("Unable to find even a single source for '" + this.toString() + "'");
}
if (list.size() > 1) {
String found = list.stream().map(c -> c.getURI().toString()).collect(Collectors.joining(","));
throw new BasicError(("Found too many sources for '" + this.toString() + "', ambiguous name. Pick from " + found));
}
return list.get(0);
}
@Override
public List<List<Content<?>>> resolveEach() {
List<List<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandSearches(prefixes, List.of(name), extensions, false);
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
List<Content<?>> contents = resolver.resolve(slotSearchPath);
resolved.add(contents);
}
}
return resolved;
}
// for testing
public LinkedHashSet<String> expandSearches() {
LinkedHashSet<String> searchSet = new LinkedHashSet<>(extensions.size()*names.size()*searchPaths.size());
for (String name : names) {
searchSet.addAll(expandSearches(name));
}
return searchSet;
return expandSearches(prefixes, names, extensions, false);
}
// for testing
public LinkedHashSet<String> expandSearches(String name) {
public LinkedHashSet<String> expandSearches(List<String> thePrefixes, List<String> names,
List<String> suffixes, boolean eachPrefix) {
LinkedHashSet<String> searchSet = new LinkedHashSet<>();
List<String> prefixesToSearch = new ArrayList<>(thePrefixes);
List<String> namesToSearch = new ArrayList<>(names);
List<String> suffixesToSearch = new ArrayList<>(suffixes);
List<String> searchPathsToTry = new ArrayList<>();
searchPathsToTry.add("");
searchPathsToTry.addAll(searchPaths);
if (prefixesToSearch.size() == 0) {
prefixesToSearch.add("");
}
if (namesToSearch.size() == 0) {
namesToSearch.add(".*");
}
if (suffixesToSearch.size() == 0) {
suffixesToSearch.add("");
}
List<String> extensionsToTry = new ArrayList<>();
extensionsToTry.add("");
extensionsToTry.addAll(extensions);
LinkedHashSet<String> searches = new LinkedHashSet<>();
for (String searchPath : searchPathsToTry) {
for (String extension : extensionsToTry) {
if (!name.endsWith(extension)) {
name = name+extension;
for (String name : namesToSearch) {
for (String suffix : suffixesToSearch) {
String search = name;
search = (search.endsWith(suffix) ? search : search + suffix);
if (eachPrefix) {
for (String prefix : prefixesToSearch) {
String withPrefix = (prefix.isEmpty() ? prefix :
prefix + FileSystems.getDefault().getSeparator())
+ search;
searches.add(withPrefix);
}
} else {
searches.add(search);
}
searchSet.add(Path.of(searchPath,name).toString());
}
}
return searchSet;
return searches;
}
// // for testing
// public LinkedHashSet<String> expandSearches(String name) {
//
// LinkedHashSet<String> searchSet = new LinkedHashSet<>();
//
// List<String> searchPathsToTry = new ArrayList<>();
// searchPathsToTry.add("");
// searchPathsToTry.addAll(prefixes);
//
// List<String> extensionsToTry = new ArrayList<>();
//// extensionsToTry.add("");
// extensionsToTry.addAll(extensions);
//
// for (String searchPath : searchPathsToTry) {
// for (String extension : extensionsToTry) {
// if (!name.endsWith(extension)) {
// name = name + extension;
// }
// searchSet.add(Path.of(searchPath, name).toString());
// }
// }
// return searchSet;
// }
@Override
public List<Content<?>> list() {
LinkedHashSet<String> searches = expandSearches();
LinkedHashSet<Content<?>> foundFiles = new LinkedHashSet<>();
// wrap in local search iterator
for (String search : searches) {
List<Content<?>> founds = resolver.resolve(search);
foundFiles.addAll(founds);
}
for (String searchPath : prefixes) {
List<Path> founds = resolver.resolveDirectory(searchPath);
FileCapture capture = new FileCapture();
for (Path path : founds) {
for (String searchPattern : searches) {
RegexPathFilter filter = new RegexPathFilter(searchPattern, true);
NBIOWalker.walkFullPath(path, capture, filter);
}
}
capture.found.stream().map(PathContent::new).forEach(foundFiles::add);
}
return new ArrayList<>(foundFiles);
}
private static String tailmatch(String name) {
if (!name.startsWith("^") && !name.startsWith(".")) {
name = ".*" + name;
}
return name;
}
private static String dotExtension(String extension) {
return extension.startsWith(".") ? extension : "." + extension;
}
// private LinkedHashSet<Pattern> expandSearchPatterns(String name) {
// LinkedHashSet<Pattern> expanded = new LinkedHashSet<>();
//
// if (extensions.size()==0) {
// expanded.add(Pattern.compile(tailmatch(name)));
// }
//
// for (String extension : extensions) {
// extension = dotExtension(extension);
// String withExtension = name.endsWith(extension) ? name : name + Pattern.quote(extension);
// withExtension=tailmatch(withExtension);
// Pattern pattern = Pattern.compile(withExtension);
// expanded.add(pattern);
// }
// return expanded;
// }
private static class RegexPathFilter implements DirectoryStream.Filter<Path> {
private final Pattern regex;
public RegexPathFilter(String pattern, boolean rightglob) {
if (rightglob && !pattern.startsWith("^") && !pattern.startsWith(".")) {
this.regex = Pattern.compile(".*" + pattern);
} else {
this.regex = Pattern.compile(pattern);
}
}
@Override
public boolean accept(Path entry) throws IOException {
String input = entry.toString();
Matcher matcher = regex.matcher(input);
boolean matches = matcher.matches();
return matches;
}
public String toString() {
return regex.toString();
}
}
private static class FileCapture implements NBIOWalker.PathVisitor, Iterable<Path> {
List<Path> found = new ArrayList<>();
@Override
public void visit(Path foundPath) {
found.add(foundPath);
}
@Override
public Iterator<Path> iterator() {
return found.iterator();
}
public String toString() {
return "FileCapture{n=" + found.size() + (found.size()>0?"," +found.get(0).toString():"") +"}";
}
}
@Override
public String toString() {
return "NBIO{" +
"resolver=" + resolver +
", prefixes=" + prefixes +
", names=" + names +
", extensions=" + extensions +
'}';
}
}

72
nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPathWalker.java → nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java
View File

@@ -1,56 +1,85 @@
package io.nosqlbench.nb.api.pathutil;
package io.nosqlbench.nb.api.content;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.List;
public class NBPathWalker {
private final static Logger logger = LogManager.getLogger(NBPathWalker.class);
public class NBIOWalker {
private final static Logger logger = LogManager.getLogger(NBIOWalker.class);
public static void walk(Path p, PathVisitor v) {
walk(p, v, NBPathWalker.WALK_ALL);
walkShortPath(p, v, NBIOWalker.WALK_ALL);
}
public static List<Path> findAll(Path p) {
Collect fileCollector = new Collect(true, false);
walk(p, fileCollector);
return fileCollector.get();
}
public static List<Path> findEndMatching(Path p, Path endingName) {
Collect fileCollector = new Collect(true, false);
MatchEnding ending = new MatchEnding(endingName.toString());
walk(p, fileCollector, ending);
return fileCollector.get();
}
public static void walk(Path p, PathVisitor v, DirectoryStream.Filter<Path> filter) {
/**
* This walks the directory structure starting at the path specified. The path visitor is invoked for every
* directory, and every non-directory which matches the filter.
* This form uses only the filename component in Paths to be matched by the filter, and the short name is also
* what is returned by the filter.
*
* @param p The path to search
* @param v The visitor to accumulate or operate on matched paths and all directories
* @param filter The Path filter to determine whether a path is included
*/
public static void walkShortPath(Path p, PathVisitor v, DirectoryStream.Filter<Path> filter) {
walk(null, p, v, filter, false);
}
/**
* This walks the directory structure starting at the path specified. The path visitor is invoked for every
* directory, and every non-directory which matches the filter.
* This form uses only the full path from the initial search path root in all Paths to be matched by
* the filter, and this form of a Path component is also returned in all Paths seen by the visitor.
*
* @param p The path to search
* @param v The visitor to accumulate or operate on matched paths and all directories
* @param filter The Path filter to determine whether a path is included
*/
public static void walkFullPath(Path p, PathVisitor v, DirectoryStream.Filter<Path> filter) {
walk(null, p, v, filter, true);
}
public static void walk(Path root, Path p, PathVisitor v, DirectoryStream.Filter<Path> filter, boolean fullpath) {
try {
FileSystemProvider provider = p.getFileSystem().provider();
DirectoryStream<Path> paths = provider.newDirectoryStream(p, (Path r) -> true);
List<Path> pathlist = new ArrayList<>();
for (Path path : paths) {
pathlist.add(path);
}
for (Path path : pathlist) {
if (fullpath && root != null) {
path = root.resolve(path);
}
if (path.getFileSystem().provider().readAttributes(path, BasicFileAttributes.class).isDirectory()) {
v.preVisitDir(path);
walk(path, v, filter);
walk(root, path, v, filter, fullpath);
v.postVisitDir(path);
} else if (filter.accept(path)) {
v.preVisitFile(path);
v.visit(path);
v.postVisitFile(path);
} else {
logger.error("error");
}
}
} catch (IOException e) {
throw new RuntimeException(e);
@@ -75,19 +104,6 @@ public class NBPathWalker {
public static DirectoryStream.Filter<Path> WALK_ALL = entry -> true;
public static class MatchEnding implements DirectoryStream.Filter<Path> {
private final String regex;
public MatchEnding(String pathEnd) {
this.regex = pathEnd;
}
@Override
public boolean accept(Path entry) throws IOException {
return entry.toString().endsWith(regex);
}
}
public static class Collect implements PathVisitor {
private final List<Path> listing = new ArrayList<>();
private final boolean collectFiles;

18
nb-api/src/main/java/io/nosqlbench/nb/api/content/PathContent.java
View File

@@ -7,6 +7,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Objects;
/**
* PathContent provides the Path-centric way of accessing
@@ -46,4 +47,21 @@ public class PathContent implements Content<Path> {
return this.path;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
PathContent that = (PathContent) o;
return Objects.equals(path, that.path);
}
@Override
public int hashCode() {
return Objects.hash(path);
}
public String toString() {
return "PathContent{" + getURI().toString() + "}";
}
}

68
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java
View File

@@ -4,9 +4,8 @@ import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.*;
import java.nio.file.spi.FileSystemProvider;
import java.util.Collections;
import java.util.Optional;
import java.util.*;
import java.util.stream.Collectors;
/**
* Resolves resources which can be found via the class loader.
@@ -23,18 +22,37 @@ public class ResolverForClasspath implements ContentResolver {
public static final ContentResolver INSTANCE = new ResolverForClasspath();
private Path resolvePath(URI uri) {
private List<Path> resolvePaths(URI uri) {
List<Path> paths = new ArrayList<>();
if (uri.getScheme() != null && !uri.getScheme().isEmpty()) {
return null;
}
URL systemResource = ClassLoader.getSystemResource(uri.getPath());
if (systemResource == null) {
// URL systemResource = ClassLoader.getSystemResource(uri.getPath());
try {
Enumeration<URL> systemResources = ClassLoader.getSystemResources(uri.getPath());
while (systemResources.hasMoreElements()) {
URL url = systemResources.nextElement();
Path p = normalize(url);
paths.add(p);
}
} catch (IOException e) {
e.printStackTrace();
}
return paths;
}
private Path normalize(URL url) {
if (url == null) {
return null;
}
URI resolved = URI.create(systemResource.toExternalForm());
URI resolved = URI.create(url.toExternalForm());
if (resolved.getScheme().equals("file")) {
return Path.of(uri.getPath());
// return Path.of(resolved.getPath());
Path current = Paths.get("").toAbsolutePath();
Path logical = Path.of(resolved.getPath());
Path relativePath = current.relativize(logical);
return relativePath;
}
FileSystem fs;
@@ -53,25 +71,27 @@ public class ResolverForClasspath implements ContentResolver {
}
@Override
public Content<?> resolve(URI uri) {
Path path = resolvePath(uri);
if (path == null) {
return null;
}
return new PathContent(path);
public List<Content<?>> resolve(URI uri) {
List<Path> paths = resolvePaths(uri);
List<Content<?>> contents = paths.stream().map(PathContent::new).collect(Collectors.toList());
return contents;
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
Path path = resolvePath(uri);
if (path == null) {
return Optional.empty();
}
if (Files.isDirectory(path)) {
return Optional.of(path);
} else {
return Optional.empty();
public List<Path> resolveDirectory(URI uri) {
List<Path> path = resolvePaths(uri);
List<Path> dirs = new ArrayList<>();
for (Path dirpath : path) {
if (Files.isDirectory(dirpath)) {
dirs.add(dirpath);
}
}
return dirs;
}
public String toString() {
return this.getClass().getSimpleName();
}
}

32
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForFilesystem.java
View File

@@ -3,6 +3,8 @@ package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class ResolverForFilesystem implements ContentResolver {
@@ -10,7 +12,7 @@ public class ResolverForFilesystem implements ContentResolver {
public static ResolverForFilesystem INSTANCE = new ResolverForFilesystem();
private Path resolvePath(URI uri) {
if (uri.getScheme()!=null&&!uri.getScheme().isEmpty()&&!uri.getScheme().equals("file")) {
if (uri.getScheme() != null && !uri.getScheme().isEmpty() && !uri.getScheme().equals("file")) {
return null;
}
Path pathFromUri = Path.of(uri.getPath());
@@ -22,25 +24,29 @@ public class ResolverForFilesystem implements ContentResolver {
}
@Override
public Content<?> resolve(URI uri) {
public List<Content<?>> resolve(URI uri) {
List<Content<?>> contents = new ArrayList<>();
Path path = resolvePath(uri);
if (path==null) {
return null;
if (path != null) {
contents.add(new PathContent(path));
}
return new PathContent(path);
return contents;
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
public List<Path> resolveDirectory(URI uri) {
List<Path> dirs = new ArrayList<>();
Path path = resolvePath(uri);
if (path == null) {
return Optional.empty();
}
if (Files.isDirectory(path)) {
return Optional.of(path);
} else {
return Optional.empty();
if (path!=null && Files.isDirectory(path)) {
dirs.add(path);
}
return dirs;
}
public String toString() {
return this.getClass().getSimpleName();
}
}

21
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForURL.java
View File

@@ -8,6 +8,9 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
public class ResolverForURL implements ContentResolver {
@@ -16,9 +19,9 @@ public class ResolverForURL implements ContentResolver {
private final static Logger logger = LoggerFactory.getLogger(ResolverForURL.class);
@Override
public Content<?> resolve(URI uri) {
public List<Content<?>> resolve(URI uri) {
if (uri.getScheme()==null) {
return null;
return List.of();
}
if (uri.getScheme().equals("http")
|| uri.getScheme().equals("https")) {
@@ -26,16 +29,20 @@ public class ResolverForURL implements ContentResolver {
URL url = uri.toURL();
InputStream inputStream = url.openStream();
logger.debug("Found accessible remote file at " + url.toString());
return new URLContent(url, inputStream);
return List.of(new URLContent(url, inputStream));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return null;
}
return List.of(); }
@Override
public Optional<Path> resolveDirectory(URI uri) {
return Optional.empty();
public List<Path> resolveDirectory(URI uri) {
return Collections.emptyList();
}
public String toString() {
return this.getClass().getSimpleName();
}
}

49
nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolver.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.commons.math3.FieldElement;
import java.net.URI;
@@ -67,32 +68,24 @@ public class URIResolver implements ContentResolver {
return this;
}
public Optional<Content<?>> resolveOptional(String uri) {
return Optional.ofNullable(resolve(uri));
}
public Content<?> resolve(String uri) {
public List<Content<?>> resolve(String uri) {
return resolve(URI.create(uri));
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
public List<Path> resolveDirectory(URI uri) {
List<Path> dirs = new ArrayList<>();
for (ContentResolver loader : loaders) {
Optional<Path> path = loader.resolveDirectory(uri);
if (path.isPresent()) {
return path;
}
dirs.addAll(loader.resolveDirectory(uri));
}
return Optional.empty();
return dirs;
}
public Content<?> resolve(URI uri) {
Content<?> resolved = null;
public List<Content<?>> resolve(URI uri) {
List<Content<?>> resolved = new ArrayList<>();
for (ContentResolver loader : loaders) {
resolved = loader.resolve(uri);
if (resolved!=null) {
break;
}
List<Content<?>> contents = loader.resolve(uri);
resolved.addAll(contents);
}
return resolved;
}
@@ -104,10 +97,7 @@ public class URIResolver implements ContentResolver {
public List<Content<?>> resolveAll(URI uri) {
List<Content<?>> allFound = new ArrayList<>();
for (ContentResolver loader : loaders) {
Content<?> found = loader.resolve(uri);
if (found!=null) {
allFound.add(found);
}
allFound.addAll(loader.resolve(uri));
}
return allFound;
}
@@ -123,4 +113,21 @@ public class URIResolver implements ContentResolver {
this.extraPaths.add(Path.of(extraPath));
return this;
}
public Content<?> resolveOne(String candidatePath) {
List<Content<?>> contents = resolveAll(candidatePath);
if (contents.size()==1) {
return contents.get(0);
}
if (contents.size()==0) {
return null;
}
throw new BasicError("Error while loading content '" + candidatePath +"', only one is allowed, but " + contents.size() + " were found");
}
public String toString() {
return "[resolver]";
}
}

17
nb-api/src/main/java/io/nosqlbench/nb/api/content/URLContent.java
View File

@@ -10,6 +10,7 @@ import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -45,6 +46,18 @@ public class URLContent implements Content<URL> {
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
URLContent that = (URLContent) o;
return Objects.equals(url, that.url);}
@Override
public int hashCode() {
return Objects.hash(url);
}
@Override
public CharBuffer getCharBuffer() {
if (buffer==null) {
@@ -62,4 +75,8 @@ public class URLContent implements Content<URL> {
public Path asPath() {
return null;
}
public String toString() {
return "URLContent{" + getURI().toString() + "}";
}
}

65
nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java
View File

@@ -8,99 +8,74 @@ import java.util.Optional;
public interface NBPathsAPI {
public static interface Facets extends
WantsSpaces, ForContentSource, ForPrefix, WantsContentName, ForName, ForExtension {}
GetSource, GetPrefix, GetName, GetExtension, DoSearch {}
public static interface WantsSpaces {
public static interface GetSource {
/**
* Only provide content from the class path and the local filesystem.
* @return this builder
*/
ForContentSource localContent();
GetPrefix localContent();
/**
* Only return content from remote URLs. If the user is providing non-URL content
* in this context, it is an error. Throw an error in that case.
* @return this builder
*/
ForContentSource remoteContent();
GetPrefix remoteContent();
/**
* Only return content from the runtime classpath, internal resources that are bundled,
* and do not return content on the file system.
* @return this builder
*/
ForContentSource internalContent();
GetPrefix internalContent();
/**
* Only return content from the filesystem, but not remote URLs nor internal bundled resources.
* @return this builder
*/
ForContentSource fileContent();
GetPrefix fileContent();
/**
* Return content from everywhere, from remote URls, or from the file system and then the internal
* bundled content if not found in the file system first.
* @return this builder
*/
ForContentSource allContent();
GetPrefix allContent();
}
public static interface ForContentSource extends ForPrefix {
public static interface GetPrefix extends GetName {
/**
* Each of the prefix paths will be searched if the resource is not found with the exact
* path given.
* @param prefixPaths A list of paths to include in the search
* @return this builder
*/
ForPrefix prefix(String... prefixPaths);
GetName prefix(String... prefixPaths);
}
public static interface ForPrefix extends WantsContentName {
/**
* Only look at exact matches of the names as given, and if not found, look for exact matches
* of the path directly within each given search directory.
* @return this builders
*/
WantsContentName exact();
/**
* Attempt {@link #exact()} matching, and if not found, also attempt to look within any
* provided search directories recursively for a path which matches the provided path at
* the end. For example "baz.csv" will be found under search directory "foo" if it is at
* "foo/bar/baz.csv", as will "bar/baz.csv", so long as "foo" is specified as a search directory.
* @return this builder
*/
WantsContentName matchtail();
/**
* Attempt {@link #exact()} matching, and if not found, search in each provided search directory
* for a path name that matches the provided name as a regex pattern.
* @return this builder
*/
WantsContentName regex();
}
public static interface WantsContentName {
public static interface GetName extends GetExtension {
/**
* Provide the names of the resources to be resolved. More than one resource may be provided.
* @param name The name of the resource to load
* @return this builder
*/
ForName name(String... name);
GetExtension name(String... name);
}
public static interface ForName extends ForExtension {
public static interface GetExtension extends DoSearch {
/**
* provide a list of optional file extensions which should be considered. If the content is
* not found under the provided name, then each of the extensios is tried in order.
* @param extensions The extension names to try
* @return this builder
*/
ForExtension extension(String... extensions);
DoSearch extension(String... extensions);
}
public static interface ForExtension {
public static interface DoSearch {
/**
* Return the result of resolving the resource.
* @return an optional {@code Content<?>} element.
@@ -112,7 +87,17 @@ public interface NBPathsAPI {
* of {@link #first()}, except that it returns a result pair-wise for each name given.
* @return A list of optional {@code Content<?>} elements.
*/
List<Optional<Content<?>>> resolveEach();
List<List<Content<?>>> resolveEach();
List<Content<?>> list();
/**
* Find exactly one source of content under the search parameters given.
* It is an error if you find none, or more than one.
* @return An optional content element.
*/
Content<?> one();
}
}

341
nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java
View File

@@ -1,341 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.nb.api.pathutil;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URI;
import java.net.URL;
import java.nio.CharBuffer;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.*;
import java.util.stream.Collectors;
public class NBPaths {
private final static Logger logger = LoggerFactory.getLogger(NBPaths.class);
public final static String DATA_DIR = "data";
/**
* <p>Look in all the provided path specifiers for an extant Path, and return
* the first one found.</p>
*
* <p>If the final character of any path specifier is the default file
* separator, then the request is for a directory. During searching,
* if a directory is found when a file is requested, or vice-versa, then
* an error is thrown withouth looking further.</p>
*
* <p>The locations that are searched include:</p>
* <OL>
* <LI>URLs. If the path specifier is a URI, then it is checked for a positive response
* before the path is returned. URLs can not be used for directories.</LI>
* <LI>The local filesystem, starting from the current directory of the process.</LI>
* <LI>The class path.</LI>
* </OL>
*
* @param pathspecs A specifier for a URL, a directory with a trailing slash, or a file
* with no trailing slash.
* @return A Path
* @throws RuntimeException if none of the specified paths is found in any of the locations
*/
public static Path findPathIn(String... pathspecs) {
Optional<Path> found = FindOptionalPathIn(pathspecs);
return found.orElseThrow();
}
public static Optional<Path> FindOptionalPathIn(String... pathspecs) {
Path foundPath = null;
for (String pathspec : pathspecs) {
if (isRemote(pathspec)) {
try {
Optional<InputStream> inputStreamForUrl = getInputStreamForUrl(pathspec);
if (inputStreamForUrl.isPresent()) {
foundPath = Path.of(URI.create(pathspec));
logger.debug("Found accessible remote file at " + foundPath.toString());
}
} catch (Exception ignored) {
}
} else {
boolean wantsADirectory = pathspec.endsWith(FileSystems.getDefault().getSeparator());
String candidatePath = wantsADirectory ? pathspec.substring(0, pathspec.length() - 1) : pathspec;
Path candidate = Path.of(candidatePath);
try {
FileSystemProvider provider = candidate.getFileSystem().provider();
provider.checkAccess(candidate, AccessMode.READ);
BasicFileAttributes attrs = provider.readAttributes(candidate, BasicFileAttributes.class);
boolean foundADirectory = attrs.isDirectory();
if (wantsADirectory != foundADirectory) {
throw new RuntimeException("for path " + pathspec + ", user wanted a " +
(wantsADirectory ? "directory" : "file") + ", but found a " +
(foundADirectory ? "directory" : "file") + " while searching paths " +
Arrays.toString(pathspecs));
}
foundPath = candidate;
} catch (Exception ignored) {
}
if (foundPath == null) {
try {
URL url = ClassLoader.getSystemResource(candidatePath);
if (url != null) {
URI uri = URI.create(url.toExternalForm());
foundPath = getPathInFilesystem(uri);
logger.debug("Found path in classpath: " + candidatePath + ": " + foundPath.toString());
}
} catch (Exception e) {
logger.trace("Error while looking in classpath for " + e.getMessage(), e);
}
}
}
}
return Optional.ofNullable(foundPath);
}
public static Optional<InputStream> getInputStreamForUrl(String path) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
}
public static CSVParser readDelimFile(String basename, char delimiter, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(delimiter).withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static Reader findRequiredReader(String basename, String extension, String... searchPaths) {
Optional<Reader> optionalReader = findOptionalReader(basename, extension, searchPaths);
return optionalReader.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static List<String> readFileLines(String basename, String... searchPaths) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "", DATA_DIR);
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
List<String> collected = buffer.lines().collect(Collectors.toList());
return collected;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static Optional<Reader> findOptionalReader(String basename, String extenion, String... searchPaths) {
return findOptionalStreamOrFile(basename, extenion, searchPaths)
.map(InputStreamReader::new)
.map(BufferedReader::new);
}
private synchronized static Path getPathInFilesystem(URI uri) {
FileSystem fileSystem = null;
try {
fileSystem = FileSystems.getFileSystem(uri);
} catch (FileSystemNotFoundException ignored) {
try {
fileSystem = FileSystems.newFileSystem(uri, new HashMap<>());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return Path.of(uri);
}
public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
Optional<InputStream> optionalStreamOrFile = findOptionalStreamOrFile(basename, extension, searchPaths);
return optionalStreamOrFile.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static Optional<InputStream> findOptionalStreamOrFile(String basename, String extension, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return stream;
}
}
return Optional.empty();
}
/**
* Search for the path
*
* @param basename Basename of path, with or without extension
* @param extension The extension of the filename
* @param searchWithin If enabled, all searchPaths are traversed, looking for a matching suffix pattern.
* @param searchPaths Additional places to look for the path suffix
* @return An optional path
*/
public static Optional<Path> findOptionalPath(String basename, String extension, boolean searchWithin, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return Optional.of(Path.of(path));
}
}
if (searchWithin) {
throw new RuntimeException("not implemented");
// for (String searchPath : searchPaths) {
// NBPathWalker.findEndMatching(Path.of(searchPath), Path.of(filename));
// }
}
return Optional.empty();
}
private static boolean isRemote(String path) {
return (path.toLowerCase().startsWith("http:")
|| path.toLowerCase().startsWith("https:"));
}
public static Optional<InputStream> getInputStream(String path) {
// URLs, if http: or https:
if (isRemote(path)) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// Files
try {
InputStream stream = new FileInputStream(path);
return Optional.of(stream);
} catch (FileNotFoundException ignored) {
}
// Classpath
ClassLoader classLoader = NBPaths.class.getClassLoader();
InputStream stream = classLoader.getResourceAsStream(path);
if (stream != null) {
return Optional.of(stream);
}
return Optional.empty();
}
public static CSVParser readFileCSV(String basename, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(',').withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static List<String> readDataFileLines(String basename) {
return readFileLines(basename, DATA_DIR);
}
public static String readFile(String basename) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "");
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
String filedata = buffer.lines().collect(Collectors.joining("\n"));
return filedata;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static CharBuffer readDataFileToCharBuffer(String basename) {
return loadFileToCharBuffer(basename, DATA_DIR);
}
public static CharBuffer loadFileToCharBuffer(String filename, String... searchPaths) {
InputStream stream = findRequiredStreamOrFile(filename, "", searchPaths);
CharBuffer linesImage;
try {
InputStreamReader isr = new InputStreamReader(stream);
linesImage = CharBuffer.allocate(1024 * 1024);
while (isr.read(linesImage) > 0) {
}
isr.close();
} catch (IOException e) {
logger.error(e.getMessage());
throw new RuntimeException(e);
}
linesImage.flip();
return linesImage.asReadOnlyBuffer();
}
}

113
nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java
View File

@@ -12,45 +12,78 @@ import static org.assertj.core.api.Assertions.assertThat;
public class NBIOTest {
@Test
public void testSimpleSearch() {
NBIO extensions = (NBIO) NBIO.all().exact().name("foo.bar");
public void testFullyQualifiedNameSearches() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testSearchPaths() {
NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").exact().name("foo.bar");
public void testExpandWildcardAndExtensionsOnly() {
NBIO extensions = (NBIO) NBIO.all().name(".*").extension("foo","bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar","act1/foo.bar","act2/foo.bar");
assertThat(searches).containsExactly(".*.foo",".*.bar");
}
@Test
public void testDefaultExtensionOn() {
NBIO extensions = (NBIO) NBIO.all().exact().name("foo.bar").extension("bar");
public void testExpandNameOnly() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension();
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().exact().name("foo.bar").extension(".bar");
}
@Test
public void testExpandNamesAndExtensions() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("baz","beez");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar.baz","foo.bar.beez");
}
@Test
public void testExpandPrefixesAndFullName() {
NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").name("foo.bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testExpandAddExtensionNotNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo.bar").extension(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandSearches();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testDefaultExtensionOff() {
NBIO extensions = (NBIO) NBIO.all().exact().name("foo").extension("bar");
public void testExpandAddExtensionNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo","foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().exact().name("foo").extension(".bar");
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo").extension(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandSearches();
assertThat(searchesDot).containsExactly("foo","foo.bar");
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testLoadCsv1() {
NBPathsAPI.ForContentSource forSourceType = NBIO.fs();
NBPathsAPI.ForPrefix forPrefix = forSourceType.prefix("nesteddir1");
NBPathsAPI.ForName forName = forPrefix.name("nesteddir2/testcsv1");
NBPathsAPI.ForExtension forCsvExtension = forName.extension(".csv");
public void testLoadCsv1Classpath() {
NBPathsAPI.GetPrefix forSourceType = NBIO.classpath();
NBPathsAPI.GetName nesteddir1 = forSourceType.prefix("nesteddir1");
NBPathsAPI.GetExtension getExtension = nesteddir1.name("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtension.extension(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
}
@Test
public void testLoadCsv1Filesystem() {
NBPathsAPI.GetPrefix forSourceType = NBIO.fs();
NBPathsAPI.GetName nesteddir1 = forSourceType.prefix("target/test-classes/nesteddir1");
NBPathsAPI.GetExtension getExtension = nesteddir1.name("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtension.extension(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
@@ -58,11 +91,51 @@ public class NBIOTest {
@Test
public void testClasspathTestResource() {
List<Optional<Content<?>>> optionals =
List<List<Content<?>>> optionals =
NBIO.classpath().name("nesteddir1/nesteddir2/testcsv12.csv").resolveEach();
assertThat(optionals).hasSize(1);
Content<?> content = optionals.get(0).get();
Content<?> content = optionals.get(0).get(0);
assertThat(content).isNotNull();
}
@Test
public void testPathSearchForExtension() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*.csv")
.extension("csv")
.list();
assertThat(list).hasSize(2);
}
@Test
public void testPathSearchForExtensionMissing() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*")
.extension("csv")
.list();
assertThat(list).hasSize(2);
}
@Test
public void testPathSearchForMultipleExtensions() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*")
.extension("csv","txt")
.list();
assertThat(list).hasSize(3);
}
@Test
public void testPathSearchForSuffix() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name("nesteddir2/testdata12")
.extension("txt")
.list();
assertThat(list).hasSize(1);
}
}

26
nb-api/src/test/java/io/nosqlbench/nb/api/pathutil/ResolverForURLTest.java
View File

@@ -8,6 +8,7 @@ import org.junit.Test;
import java.net.URL;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@@ -16,9 +17,9 @@ public class ResolverForURLTest {
@Test
public void testUrlResource() {
ResolverForURL r = new ResolverForURL();
Content<?> c = r.resolve("http://google.com");
List<Content<?>> c = r.resolve("http://google.com");
assertThat(c).isNotNull();
Object location = c.getLocation();
Object location = c.get(0).getLocation();
assertThat(location).isInstanceOf(URL.class);
assertThat(location.toString()).isEqualTo("http://google.com");
}
@@ -27,30 +28,31 @@ public class ResolverForURLTest {
public void testFileResource() {
String p = "src/test/resources/nesteddir1/nesteddir2/testcsv12.csv";
ResolverForFilesystem r = new ResolverForFilesystem();
Content<?> c = r.resolve(p);
List<Content<?>> c = r.resolve(p);
assertThat(c).isNotNull();
Object location = c.getLocation();
Object location = c.get(0).getLocation();
assertThat(location).isInstanceOf(Path.class);
assertThat(location.toString()).isEqualTo(p);
String q = "nesteddir1/nesteddir2/testcsv12.csv";
Content<?> notfound = r.resolve(q);
assertThat(notfound).isNull();
List<Content<?>> notfound = r.resolve(q);
assertThat(notfound).isEmpty();
}
@Test
public void testCPResource() {
String p = "nesteddir1/nesteddir2/testcsv12.csv";
String cprootForTestContext = "target/test-classes/";
String resourcePathWithinClasspathRoots = "nesteddir1/nesteddir2/testcsv12.csv";
ResolverForClasspath r = new ResolverForClasspath();
Content<?> c = r.resolve(p);
List<Content<?>> c = r.resolve(resourcePathWithinClasspathRoots);
assertThat(c).isNotNull();
Object location = c.getLocation();
assertThat(location.toString()).isEqualTo(p);
Object location = c.get(0).getLocation();
assertThat(location.toString()).isEqualTo(cprootForTestContext + resourcePathWithinClasspathRoots);
String q = "src/test/resources/nesteddir1/nesteddir2/testcsv12.csv";
Content<?> notfound = r.resolve(q);
assertThat(notfound).isNull();
List<Content<?>> notfound = r.resolve(q);
assertThat(notfound).isEmpty();
}
}

3
nb-api/src/test/resources/nesteddir1/nesteddir2/testdata12.txt Normal file
View File

@@ -0,0 +1,3 @@
heading1, heading2
row1col1, row1col2
"row2col1",row2col2